{ "Disclaimer": [ "The information in this database is for general guidance and is not to be relied upon as professional advice.", "DSIT has tried to ensure that the information on this database is accurate and up to date. DSIT will not accept liability for any loss and/or damage or inconvenience arising as a consequence of any use of or the inability to use any information on this website. DSIT endeavours to provide a reliable service; DSIT does not guarantee that its service will be uninterrupted or error-free. DSIT shall not be responsible for claims brought by third parties arising from your use of this database.", "DSIT assumes no responsibility for the contents of linked websites. The inclusion of any link should not be taken as endorsement of any kind by DSIT of the linked website or any association with its operators. DSIT has no control over the availability of the linked pages." ], "Copyright": "The copyright of the original material remains that of the original authors and any usage of excerpts in the mapping is made under fair use. References to organisations do not imply endorsement by DSIT.", "Version": "1.0", "Data": { "Principle 1": { "Central Bank of the UAE": { "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E": { "link": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirements": [ { "requirementID": "6. Continuous Monitoring and Review - f", "requirementText": "LFIs should ensure they have systems in place to keep up to date with legal, third-party provider and market developments with respect to the use of AI. " } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirements": [ { "requirementID": "1.3.1 Educate Personnel on AI - Training OT teams", "requirementText": "Training OT teams on AI fundamentals and threat modeling so teams can effectively interpret and validate AI outputs and maintain operational competencies alongside AI systems—for example, training teams to use alternative sensors (e.g., human senses, vibration or temperature sensors, voltage readings) for validating AI output—and know what actions to take if AI outputs are invalid." } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "BCR-05", "requirementText": "Develop, identify, and acquire documentation, both internally and from external parties, that is relevant to support the business continuity and operational resilience programs. Make the documentation available to authorized stakeholders and review at least annually or upon significant changes." }, { "requirementID": "GRC-07", "requirementText": "Identify and document all relevant standards, regulations, legal/contractual, and statutory requirements, which are applicable to your organization. Review at least annually or when a substantial change occurs within the organization." }, { "requirementID": "GRC-08", "requirementText": "Establish and maintain contact with related special interest groups and other relevant entities in line with business context." }, { "requirementID": "HRS-11", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain a security awareness training program for all employees of the organization and provide regular training updates." }, { "requirementID": "HRS-13", "requirementText": "Make employees aware of their roles and responsibilities for maintaining awareness and compliance with established policies and procedures and applicable legal, statutory, or regulatory compliance obligations." }, { "requirementID": "HRS-14", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures defining the AI training program for all relevant personnel of the organization based on their roles and provide regular training updates." }, { "requirementID": "HRS-15", "requirementText": "Establish, document, and communicate to all personnel the policies and procedures on the acceptable use of AI technologies within the organization." } ] } }, "CoSAI": { "AI Incident Response Framework": { "link": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirements": [ { "requirementID": "3.3.1. Preparation Phase - Response Capability Development", "requirementText": "• Specialized AI security training\n• Response tools for AI systems\n• Emergency access procedures\n• Backup procedures\n• Recovery testing" } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "2.5.1", "requirementText": "The entity shall promote awareness and understanding of potential AI/ML attacks, fostering a proactive security culture through education, training, and preparedness for effective incident response." }, { "requirementID": "3.1.1 Cyber Security Policies & Procedures - 4", "requirementText": "AI/ML cyber security policies and procedures should be communicated and made accessible to all relevant personnel, ensuring awareness and understanding." }, { "requirementID": "3.1.1 Cyber Security Policies & Procedures - 5", "requirementText": "AI/ML cyber security policies and procedures should be reviewed and updated regularly to address emerging threats, technological advancements, and changes in legal and regulatory requirements relevant to AI/ML." }, { "requirementID": "3.2.5 Application Security - 5", "requirementText": "The entity should provide training to developers on secure coding practices and common security threats in AI/ML application development." }, { "requirementID": "3.4.1 Context-Specific AI/ML Security - 5", "requirementText": "The entity should provide comprehensive cyber security training to AI/ML operators (if any), equipping them with the necessary knowledge and skills to operate the AI/ML system securely in its application context." }, { "requirementID": "3.5.1 Understanding and Anticipating AI/ML Attacks - 1", "requirementText": "The entity should actively work to understand potential AI/ML attacks, including their methods, implications, and possible mitigations, to anticipate and prepare for such events." }, { "requirementID": "3.5.1 Understanding and Anticipating AI/ML Attacks - 2", "requirementText": "The entity should be aware of and educated about common types of AI/ML attacks, such as adversarial attacks, poisoning attacks, and model inversion attacks." }, { "requirementID": "3.5.1 Understanding and Anticipating AI/ML Attacks - 3", "requirementText": "The entity should engage in threat intelligence and information sharing with other organizations, research institutions, and authorities to stay updated about the latest trends and developments in AI/ML attacks." }, { "requirementID": "3.5.1 Understanding and Anticipating AI/ML Attacks - 4", "requirementText": "The entity should implement user training and awareness programs to ensure that staff and stakeholders understand the risks associated with AI/ML systems and are equipped to recognize and respond to potential threats." } ] } }, "Databricks": { "The Databricks AI Security Framework": { "link": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirements": [ { "requirementID": "DASF 59: Use clean rooms", "requirementText": "Building AI applications today necessitates collaborative efforts across organizations and teams, emphasizing a commitment to privacy and data security. Databricks Clean Rooms offer a secure environment for private collaboration on diverse data and AI tasks, spanning machine learning, SQL queries, Python, R and more. Designed to facilitate seamless collaboration across different cloud and data platforms, Databricks Clean Rooms ensure multi party collaboration without compromising data privacy or security and enables organizations to build scalable AI applications in a privacy-safe manner." }, { "requirementID": "DASF 61: Train users on AI risk taxonomy and AI/ML security", "requirementText": "Provide secure coding education and AI vulnerability awareness for model developers. Training personnel who manage AI infrastructure on cybersecurity best practices is essential to prevent human errors that could lead to security breaches.\nEstablish a risk taxonomy that categorizes risks within harmful, out-of-scope, and hallucinated outputs, tool calls, and other risks based on application-specific usage. \nhttps://www.databricks.com/trust/responsibleAI\nhttps://www.databricks.com/trust/ai-security" } ] } }, "ENISA": { "Multilayer Framework for Good Cybersecurity Practices for AI": { "link": "https://www.enisa.europa.eu/sites/default/files/publications/Multilayer%20Framework%20for%20Good%20Cybersecurity%20Practices%20for%20AI.pdf", "requirements": [ { "requirementID": "Human Capital 1", "requirementText": "Have you built/do you plan to build synergies with educational authorities/institutions to increase AI cybersecurity capabilities at all levels of education?" }, { "requirementID": "Human Capital 2", "requirementText": "Do you offer awareness campaigns about the secure development and use of AI solutions?" }, { "requirementID": "Human Capital 3", "requirementText": "Do you provide guidance and best practices on how to improve AI security?" }, { "requirementID": "Human Capital 4", "requirementText": "Do you consider AI cybersecurity in syllabus of courses dedicated to AI or to CS?" }, { "requirementID": "Human Capital 5", "requirementText": "Do you offer practical trainings to the AI stakeholders and can elaborate to which stakeholders?" }, { "requirementID": "Human Capital 6", "requirementText": "Do you organise national, regional and cross-border cybersecurity exercises enabling the upskilling of the AI stakeholders?" }, { "requirementID": "From the lab to the market 1", "requirementText": "What type of support (funding/scholarships/ collaboration opportunities) do you offer to increase the cybersecurity capabilities of newly innovative solutions that rely on AI?" }, { "requirementID": "From the lab to the market 5", "requirementText": "Have you informed national AI stakeholders on cybersecurity requirements set by the NCAs for their AI products and how do you do it?" }, { "requirementID": "Networking 4", "requirementText": "Do you promote/inform about new initiatives on AI security and vulnerabilities sharing? Like a catalogue of pointers to initiatives (e.g. NIST AI framework)?" } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.1.1-1", "requirementText": "Organizations' cyber security training programme shall include AI security content which shall be regularly reviewed and updated, such as if new substantial AI-related security threats emerge." }, { "requirementID": "Provision 5.1.1-1.1", "requirementText": "AI security training shall be tailored to the specific roles and responsibilities of staff members." }, { "requirementID": "Provision 5.1.1-2", "requirementText": "As part of an Organization's wider staff training programme, they shall require all staff to maintain awareness of the latest security threats and vulnerabilities that are AI-related. Where available, this awareness shall include proposed mitigations." }, { "requirementID": "Provision 5.1.1-2.1", "requirementText": "These updates should be communicated through multiple channels, such as security bulletins, newsletters, or internal knowledge-sharing platforms. This will ensure broad dissemination and understanding among the staff." }, { "requirementID": "Provision 5.1.1-2.2", "requirementText": "Organizations shall provide developers with training in secure coding and system design techniques specific to AI development, with a focus on preventing and mitigating security vulnerabilities in AI algorithms, models, and associated software." } ] }, "SAI 002 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_gr/SAI/001_099/002/01.01.01_60/gr_SAI002v010101p.pdf", "requirements": [ { "requirementID": "6.1.2 Cybersecurity hygiene - 1", "requirementText": "Phishing attacks are a common attack vector for malicious actors seeking to gain credentials or access to a system. Good training and employee awareness remain the best defence against this kind of attack." } ] }, "TR 104 048 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_tr/104000_104099/104048/01.01.01_60/tr_104048v010101p.pdf", "requirements": [ { "requirementID": "6.1.2 Cybersecurity hygiene - 1", "requirementText": "Phishing attacks are a common attack vector for malicious actors seeking to gain credentials or access to a system. Good training and employee awareness remain the best defence against this kind of attack." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.1.1-1", "requirementText": "\"Organizations' cyber security training programme shall include AI security content which shall be regularly reviewed and updated where necessary, such as if new substantial AI-related security threats emerge.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nAI attack types are still being understood and evolving so staff can be unaware of unique AI vulnerabilities like data poisoning, adversarial attacks, or prompt injections, leaving the system exposed to sophisticated attacks.\n\nExample Measures/Controls:\nEstablish an AI Security Awareness Training Programme that covers basic AI concepts, threats, applicable regulations, etc. Include guidance on how to monitor for threats and the escalation paths for reporting security concerns." }, { "requirementID": "Provision 5.1.1-1.1", "requirementText": "\"AI security training shall be tailored to the specific roles and responsibilities of staff members.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout tailored AI security training, staff can lack the knowledge to address role-specific risks, leading to ineffective implementation of security measures, increased vulnerability to threats, and potential misuse or mismanagement of AI systems.\n\nExample Measures/Controls 1:\nRole-Specific AI Security Training: Provide role-specific AI security training tailored to the responsibilities of each\nstaff category\n\nExample Measures/Controls 2:\nIncorporate Training on AI Threat Modelling and Red Teaming. Provide developers and other technical staff with\ntraining on threat modelling techniques and red teaming techniques tailored for AI." }, { "requirementID": "Provision 5.1.1-2", "requirementText": "\"As part of an Organization's wider staff training programme, they shall require all staff to maintain awareness of the latest security threats and vulnerabilities that are AI-related. Where available, this awareness shall include proposed mitigations.\" (ETSI TS 104 223 [i.1]) \n\nRelated threats/risks:\nAI systems face evolving threats, and staff who are not updated regularly on these vulnerabilities can unknowingly expose systems to risks, such as adversarial attacks or personal data leaks which will be in breach of data protection regulations. \n\nExample Measures/Controls:\nMaintain training awareness: Update training material regularly with new examples of AI threats (e.g. prompt injections, adversarial attacks) and mitigation techniques." }, { "requirementID": "Provision 5.1.1-2.1", "requirementText": "\"These updates should be communicated through multiple channels, such as security bulletins, newsletters, or internal knowledge-sharing platforms. This will ensure broad dissemination and understanding among the staff.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nFailure to communicate new developments through diverse channels can result in uneven dissemination of critical security information, leaving some staff unaware of vulnerabilities, mitigations, or best practices, increasing the risk of oversight and security lapses.\n\nExample Measures/Controls:\nDisseminate Regular Security Updates and Bulletins." }, { "requirementID": "Provision 5.1.1-2.2", "requirementText": "\"Organizations shall provide developers with training in secure coding and system design techniques specific to AI development, with a focus on preventing and mitigating security vulnerabilities in AI algorithms, models, and associated software.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout specialized training in secure coding and AI system design, developers can inadvertently introduce vulnerabilities into AI algorithms, models, or supporting software, increasing the risk of exploits, data breaches, or system failures.\n\nExample Measures/Controls:\nProvide secure coding training for engineers related to AI threats and incorporating guidelines from OWASP, NCSC, the ETSI Mitigation Strategy report [i.2]." } ] } }, "European Commission": { "Ethics guidelines for trustworthy AI": { "link": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirements": [ { "requirementID": "2.2.6 Education and Awareness to Foster an Ethical Mind-Set", "requirementText": "Trustworthy AI encourages the informed participation of all stakeholders. Communication, education and training play an important role, both to ensure that knowledge of the potential impact of AI systems is widespread, and to make people aware that they can participate in shaping the societal development. This includes all stakeholders, e.g. those involved in making the products (the designers and developers), the users (companies or individuals) and other impacted groups (those who may not purchase or use an AI system but for whom decisions are made by an AI system, and society at large). Basic AI literacy should be fostered across society. A prerequisite for educating the public is to ensure the proper skills and training of ethicists in this space." } ] } }, "Google": { "Secure AI Framework": { "link": "https://www.saif.google/secure-ai-framework", "requirements": [ { "requirementID": "Internal Policies and Education", "requirementText": "Publish comprehensive AI security and privacy policies and education for your employees." } ] } }, "ICO": { "Guidance on the AI Auditing Framework - Draft guidance for consultation ": { "link": "https://ico.org.uk/media2/about-the-ico/consultations/2617219/guidance-on-the-ai-auditing-framework-draft-for-consultation.pdf", "requirements": [ { "requirementID": "How should we approach AI governance and risk management?", "requirementText": "While AI increases the importance of embedding data protection by design and default into an organisation’s culture and processes, the technical complexities of AI systems can make this more difficult. Demonstrating how you have addressed these complexities is an important element of accountability.\n\nYou cannot delegate these issues to data scientists or engineering teams. Your senior management, including Data Protection Officers (DPOs), are accountable for understanding and addressing them appropriately and promptly.\n\nTo do so, in addition to their own upskilling, they need diverse, wellresourced, teams to support them in discharging their responsibilities. You also need to align your internal structures, roles and responsibilities maps, training requirements, policies and incentives to your overall AI governance and risk management strategy. " }, { "requirementID": "Preventative Controls - 1", "requirementText": "Subscribe to security advisories to receive alerts of vulnerabilities." }, { "requirementID": "Preventative Controls - 12", "requirementText": "Have processes in place to review the latest privacy enhancing techniques, assess the technique's applicability to their context, and implement it where appropriate." }, { "requirementID": "Preventative Controls - 15", "requirementText": "Ensure staff are trained to understand the breach reporting policy and what procedures to follow." } ] } }, "IMDA": { "Model AI Governance Framework for Agentic AI": { "link": "https://www.imda.gov.sg/-/media/imda/files/about/emerging-tech-and-research/artificial-intelligence/mgf-for-agentic-ai.pdf", "requirements": [ { "requirementID": "2.4.3 Users who integrate agents into their work processes", "requirementText": "Such users typically utilise agents as part of their internal workflows e.g. coding assistants, automation of enterprise processes. The agent acts for and on behalf of the user.\nFor these users, in addition to the information in the previous section, layer on education and training so that users can use the agents responsibly. Key aspects include education and training on:\n• Foundational knowledge on agents\no Relevant use cases, so that the users understand how to best integrate the agents into their day-to-day work, and the scenarios under which the use of agents should be restricted (e.g. do not use an agent for confidential data)\no Instructing the agents e.g. general best practices in prompting, glossary of keywords to elicit specific responses\no Agents’ range of actions, so that the user is aware of their capabilities and potential impact\n• Effective oversight of agents\no Common agent failure modes, such as hallucinations, getting stuck in loops after errors, so that the user can identify and flag out issues.\no Ongoing support, such as regular refreshers to update users on latest features and common user mistakes\n• Potential impact on tradecraft\no As agents take over entry level tasks, which typically serve as the training ground for new staff, this could lead to loss of basic operational knowledge for the users.\no Organisations should identify core capabilities of each job and provide sufficient training and work exposure so that users retain foundational skills." } ] } }, "ISO": { "42001:2023 - Information technology — Artificial intelligence — Management system": { "link": "https://www.iso.org/standard/42001", "requirements": [ { "requirementID": "5.2", "requirementText": "AI policy" } ] } }, "ISO/IEC": { "TR 27091": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:27091:dis:ed-1:v1:en", "requirements": [ { "requirementID": "6.4", "requirementText": "Guidance on AI models" } ] }, "TR 27563:2023": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:tr:27563:ed-1:v1:en", "requirements": [ { "requirementID": "7.1", "requirementText": "Describe ecosystem" } ] } }, "METI (Japan)": { "Governance Guidelines for Implementation of AI Principles": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20220128_2.pdf", "requirements": [ { "requirementID": "Action Target 1-1", "requirementText": "Companies that develop and operate AI systems should, under the leadership of top management, understand not only positive impacts but also negative impacts, including unintended risks, that AI systems may have. This information should be reported to the top management and shared among those in top managerial positions, and their understanding should be updated in a timely manner." }, { "requirementID": "Action Target 1-2", "requirementText": "Companies that develop and operate AI systems should, under the leadership of top management, understand the current state of social acceptance based on opinions of not only direct stakeholders, but potential stakeholders before full-scale provision of the AI systems. In addition, even after the full-scale operation, companies should obtain opinions of stakeholders again and update their perspectives in a timely manner." }, { "requirementID": "Action Target 3-2", "requirementText": "Companies that develop and operate AI systems should, under the leadership of top management, strategically improve AI literacy in order to properly operate their AI management system, considering outside learning materials as an option. For example, this may include education to boost general literacy on AI ethics for those in the top management, management teams, and those in operations positions responsible for legal and ethical aspects of AI system development and operation, as well as training for those responsible for AI system development and operation not only on AI ethics but also on AI technology. Companies that provide data should take steps to improve the general literacy of AI ethics of employees engaged in data provision by referring to practical examples for AI system developers and operators." }, { "requirementID": "Action Target 3-3-2", "requirementText": "Companies that develop and operate AI systems should, under the leadership of top management, regularly collect relevant information such as formulation of rules for the development and operation of AI systems, best practice, and incidents, and encourage the exchange of views within and outside the company." }, { "requirementID": "Action Target 4-3", "requirementText": "Companies that develop and operate AI systems should consider ranking information relevant to AI governance such as one related to AI governance goal setting and establishment and operation of AI management systems as non-financial information in the Corporate Governance Code and proactively disclosing such information. Non-listed companies should also consider proactively disclosing information related to AI governance activities. If companies decide not to disclose such information after due consideration, they should be prepared to explain the reason externally." } ] } }, "MIC/METI (Japan)": { "AI Guidelines for Business": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirements": [ { "requirementID": "Education/literacy - 1", "requirementText": "Take the necessary steps to ensure that the persons engaged in AI in each AI business actor acquire AI literacy of the level sufficient for the engagement." }, { "requirementID": "Education/literacy - 2 (b)", "requirementText": "Provide educational opportunities taking into account differences in knowledge and skills among generations so that various people can acquire a deeper understanding of benefits of AI and enhance the resilience against risks." } ] } }, "Microsoft": { "Cloud Adoption Framework - Secure AI": { "link": "https://learn.microsoft.com/en-us/azure/cloud-adoption-framework/scenarios/ai/secure", "requirements": [ { "requirementID": "Discover AI security risks\n1 - Identify AI system risks across your environment", "requirementText": "AI adoption introduces classes of risk that may not be explicitly addressed in traditional threat modeling. While your existing enterprise framework may already be capable of modeling these risks, AI governance requires deliberate validation that it does so. Begin with your established threat modeling framework, such as STRIDE. Then reference AI‑specific risk inventories, like MITRE ATLAS and OWASP Generative AI risk to confirm that AI‑specific attack techniques, misuse scenarios, and systemic risks are adequately represented. Use these sources to supplement, not replace, your existing framework and to inform consistent AI risk identification across the organization." }, { "requirementID": "Discover AI security risks\n2 - Assess AI data risks throughout your workflows", "requirementText": "Sensitive data in AI workflows increases the risk of insider threats and data leaks that can compromise business operations. Data risk assessment helps you prioritize security investments based on actual exposure levels. Use tools like Microsoft Purview Insider Risk Management to assess enterprise-wide data risks and prioritize them based on data sensitivity levels." } ] } }, "MITRE": { "ATLAS Framework": { "link": "https://atlas.mitre.org/mitigations", "requirements": [ { "requirementID": "AML.M0018 - User Training", "requirementText": "Educate AI model developers to on AI supply chain risks and potentially malicious AI artifacts. Educate users on how to identify deepfakes and phishing attempts." } ] } }, "Multi Agency": { "Guidelines for secure AI system development": { "link": "https://www.ncsc.gov.uk/files/Guidelines-for-secure-AI-system-development.pdf", "requirements": [ { "requirementID": "Raise staff awareness of threats and risks", "requirementText": "System owners and senior leaders understand threats to secure AI and their mitigations. Your data scientists and developers maintain an awareness of relevant security threats and failure modes and help risk owners to make informed decisions. You provide users with guidance on the unique security risks facing AI systems (for example, as part of standard InfoSec training) and train developers in secure coding techniques and secure and responsible AI practices." } ] } }, "NIST": { "AI 800-1": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirements": [ { "requirementID": "Practice 1.1: Anticipate model capabilities - 1", "requirementText": "Prioritize evidence from widely deployed and studied proxy models." }, { "requirementID": "Practice 2.2 Establish an organizational plan to manage misuse risk - 5", "requirementText": "Consider consulting existing resources for developing and implementing such organizational plans, such as the NIST RMF and other guidance." } ] }, "AI RMF 1.0": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirements": [ { "requirementID": "GOVERN 1.1", "requirementText": "Legal and regulatory requirements involving AI are understood, managed, and documented." }, { "requirementID": "GOVERN 1.2", "requirementText": "The characteristics of trustworthy AI are integrated into organizational policies, processes, procedures, and practices." }, { "requirementID": "GOVERN 2.2", "requirementText": "The organization’s personnel and partners receive AI risk management training to enable them to perform their duties and responsibilities consistent with related policies, procedures, and agreements." }, { "requirementID": "GOVERN 4.1", "requirementText": "Organizational policies and practices are in place to foster a critical thinking and safety-first mindset in the design, development, deployment, and uses of AI systems to minimize potential negative impacts." }, { "requirementID": "MAP 3.4", "requirementText": "Processes for operator and practitioner proficiency with AI system performance and trustworthiness – and relevant technical standards and certifications – are defined, assessed, and documented." } ] }, "IR 8596: Cybersecurity Framework Profile for Artificial Intelligence (Cyber AI Profile): NIST Community Profile": { "link": "https://csrc.nist.gov/pubs/ir/8596/iprd", "requirements": [ { "requirementID": "GV.OC-02", "requirementText": "Internal and external stakeholders are understood, and their needs and expectations regarding cybersecurity risk management are understood and considered" }, { "requirementID": "GV.OC-03", "requirementText": "Legal, regulatory, and contractual requirements regarding cybersecurity— including privacy and civil liberties obligations—are understood and managed" }, { "requirementID": "GV.OC-05", "requirementText": "Outcomes, capabilities, and services that the organization depends on are understood and communicated" }, { "requirementID": "GV.RR-04", "requirementText": "Cybersecurity is included in human resources practices" }, { "requirementID": "PR.AT-01", "requirementText": "Personnel are provided with awareness and training so that they possess the knowledge and skills to perform general tasks with cybersecurity risks in mind" }, { "requirementID": "PR.AT-02", "requirementText": "Individuals in specialized roles are provided with awareness and training so that they possess the knowledge and skills to perform relevant tasks with cybersecurity risks in mind" } ] }, "SP 800-218A": { "link": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirements": [ { "requirementID": "PO.2.2", "requirementText": "Provide role-based training for all personnel with responsibilities that contribute to secure development. Periodically review personnel proficiency and role-based training, and update the training as needed.\n\nRole-based training should include understanding cybersecurity vulnerabilities and threats to AI models and their possible mitigations." }, { "requirementID": "PW.5.1", "requirementText": "Follow all secure coding practices that are appropriate to the development languages and environment to meet the organization’s requirements." }, { "requirementID": "PW.9.1", "requirementText": "Define a secure baseline by determining how to configure each setting that has an effect on security or a securityrelated setting so that the default settings are secure and do not weaken the security functions provided by the platform, network infrastructure, or services." }, { "requirementID": "RV.3.1", "requirementText": "Analyze identified vulnerabilities to determine their root causes." }, { "requirementID": "RV.3.2", "requirementText": "Analyze the root causes over time to identify patterns, such as a particular secure coding practice not being followed consistently." } ] } }, "OWASP": { "AI Exchange": { "link": "https://owaspai.org/docs/ai_security_overview/", "requirements": [ { "requirementID": "1.1 General governance controls - SEC PROGRAM", "requirementText": "Security Program: Make sure the organization has a security program (also referred to as information security management system) and that it includes the whole AI lifecycle and AI specific aspects." }, { "requirementID": "1.1 General governance controls - CHECK COMPLIANCE", "requirementText": "Check compliance: Make sure that AI-relevant laws and regulations are taken into account in compliance management (including security aspects). If personal data is involved and/or AI is applied to make decisions about individuals, then privacy laws and regulations are also in scope. See the Privacy section for details." }, { "requirementID": "1.1 General governance controls - SEC EDUCATE", "requirementText": "Security education for data scientists and development teams on AI threat awareness, including attacks on models. It is essential for all engineers, including data scientists, to attain a security mindset." } ] }, "OWASP Model Context Protocol (MCP) Top 10": { "link": "https://owasp.org/www-project-mcp-top-10/", "requirements": [ { "requirementID": "MCP09:2025 – Shadow MCP Servers - 6", "requirementText": "Security Awareness & Developer Education\n- Conduct regular security workshops explaining the risks of shadow MCP deployments.\n- Encourage teams to use sandboxed, approved experimentation zones with pre-hardened MCP templates.\n- Include MCP registration requirements in development onboarding documentation." } ] } }, "Personal Data Protection Commission Singapore (PDPC)": { "Model Artificial Intelligence Governance Framework Second Edition": { "link": "https://www.pdpc.gov.sg/-/media/files/pdpc/pdf-files/resource-for-organisation/ai/sgmodelaigovframework2.pdf", "requirements": [ { "requirementID": "1. Clear roles and responsibilities for the ethical deployment of AI - b)", "requirementText": "Personnel and/or departments having internal AI governance functions should be fully aware of their roles and responsibilities, be properly trained, and be provided with the resources and guidance needed for them to discharge their duties." }, { "requirementID": "1. Clear roles and responsibilities for the ethical deployment of AI - c) (iv)", "requirementText": "Ensuring relevant staff dealing with AI systems are properly trained. Where applicable and necessary, staff who are working and interacting directly with AI models may need to be trained to interpret AI model output and decisions and to detect and manage bias in data. Other staff whose work deals with the AI system (e.g. a customer relationship officer answering customer queries about the AI system, or a salesperson using an AI-enabled product to make a recommendation) should be trained to be at least aware of and sensitive to the benefits, risks and limitations when using AI, so that they know when to alert subject-matter experts within their organisations." }, { "requirementID": "2. Risk management and internal controls - b) (iii)", "requirementText": "Ensuring proper knowledge transfer whenever there are changes in key personnel involved in AI activities. This will reduce the risk of staff movement creating a gap in internal governance." }, { "requirementID": "Data for Model Development - b) Ensuring data quality", "requirementText": "Organisations are encouraged to understand and address factors that may affect the quality of data, such as:\ni. The accuracy of the dataset, in terms of how well the values in the dataset match the true characteristics of the entities described by the dataset;\nii. The completeness of the dataset, both in terms of attributes and items;\niii. The veracity of the dataset, which refers to how credible the data is, including whether the data originated from a reliable source;\niv. How recently the dataset was compiled or updated;\nv. The relevance of the dataset and the context for data collection, as it may affect the interpretation of and reliance on the data for the intended purpose;\nvi. The integrity of the dataset that has been joined from multiple datasets, which refers to how well extraction and transformation have been performed;\nvii. The usability of the dataset, including how well the dataset is structured in a machineunderstandable form; and\nviii. Human interventions (e.g. if any human has filtered, applied labels, or edited the data)." } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirements": [ { "requirementID": "8.1", "requirementText": "An Entity must ensure that the functions associated with Al governance are fully aware of their roles and responsibilities, properly trained, and provided with the resources and guidance needed for them to discharge their duties." }, { "requirementID": "8.2.5", "requirementText": "Ensure that all staff who deal with the Al System are aware of and sensitive to the benefits, risks and limitations of using Al." }, { "requirementID": "13.8.1", "requirementText": "An Entity must have Operators who are trained to be able to interpret Al generated output where that output is of a nature that can be used by Operators to make decisions." } ] } }, "Smart Dubai (UAE)": { "AI Ethics Principles & Guidelines": { "link": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirements": [ { "requirementID": "1.2.6.2", "requirementText": "Development of AI systems informing significant decisions should include consultation with experts in the field in which the system will be deployed." } ] } } }, "Principle 2": { "CEN/CENELEC": { "prEN 40000-1-1": { "link": "https://genorma.com/en/standards/pren-40000-1-1", "requirements": [ { "requirementID": "product control", "requirementText": "a measure on a product that modifies risk." } ] }, "prEN 40000-1-2: Cybersecurity requirements for products with digital elements - Part 1-2: Principles for cyber resilience": { "link": "https://genorma.com/en/standards/pren-40000-1-2", "requirements": [ { "requirementID": "5.3", "requirementText": "Security by Design" }, { "requirementID": "5.4", "requirementText": "Secure by Default" }, { "requirementID": "6.2", "requirementText": "Product Context" }, { "requirementID": "7.2", "requirementText": "Product cybersecurity planning" }, { "requirementID": "7.3", "requirementText": "Product cybersecurity requirements" }, { "requirementID": "7.11", "requirementText": "Third-party component cybersecurity management" } ] } }, "Central Bank of the UAE": { "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E": { "link": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirements": [ { "requirementID": "5. Data Quality, Privacy and Security - c", "requirementText": "Institutions should incorporate privacy-by-design and security-by-design principles into AI systems and maintain safeguards to protect data from unauthorised access or misuse. " }, { "requirementID": "7. Human Oversight and Consumer Protection - d", "requirementText": "The design and deployment of AI and ML systems should promote fair and equitable treatment. AI should not be used to target consumers with unsuitable products or to engage in pressure-selling or misleading marketing. Institutions should ensure that promotional materials and chatbots comply with disclosure requirements, all in line with the Consumer Protection Regulation’s emphasis on good disclosure. " } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirements": [ { "requirementID": "1.2.1 Understand the Secure AI System Development Lifecycle", "requirementText": "Design the AI system with security considerations in mind from its inception, including using robust coding, protocols, and data protection measures." }, { "requirementID": "1.3.2 Educate Personnel on AI - Developing Clear Standard Operating Procedures", "requirementText": "Developing clear standard operating procedures (SOPs) for all operations (including AI-related operations), interventions, and incidents to support stakeholder awareness of their roles and responsibilities in managing AI-enabled OT systems." }, { "requirementID": "2.1 Consider the OT Business Case for OT", "requirementText": "Before incorporating an AI system into their OT environment, critical infrastructure owners and operators should assess if AI technologies are the most appropriate solution for their specific needs and requirements compared to other technologies. Critical infrastructure owners and operators should further consider whether an established capability meets their needs before pursuing more complex and novel AI enabled solutions. While AI comes with unique benefits, it is an evolving technology that requires continuous evaluation of risks." }, { "requirementID": "2.2.7 - Prioritising OT Data Protection", "requirementText": "Prioritize the protection of critical types of OT data, including the following: Engineering Configuration Data. These include network diagrams, asset inventories, documentation on operations sequences, safety-related information, logic diagrams, and schematics. These data points have enduring value and are highly valuable to cyber adversaries. Ephemeral OT Data. Data from industrial measurement technology, especially process measurement technology (e.g., voltage or temperature, pressure levels, mass/volume flow rates) can provide insight into organizational activities or system behavior. If that data is used to train or update an AI model, the data may become accessible or stored (statistically) for a longer period of time in the model. As such, securing these data points can be important for protecting intellectual property (IP) and patterns of activity" }, { "requirementID": "2.4.2", "requirementText": "Integrate AI systems into their overall security and cybersecurity framework (see 3.2 Integrating AI Into Existing Security and Cybersecurity Frameworks)." }, { "requirementID": "2.4.3", "requirementText": "Add AI security considerations to a comprehensive security strategy that also includes traditional cybersecurity considerations, such as data encryption, access controls, and intrusion detection systems. Critical infrastructure owners and operators should define and validate security clauses in cloud contracts, explicitly outlining any AI security responsibilities, compliance standards, and support provisions, including data protection, access controls, incident response, and audit capabilities. o Cloud providers should provide detailed documentation that outlines security obligations specific to AI capabilities, in addition to traditional cloud security shared responsibility models" }, { "requirementID": "2.4.4", "requirementText": "Consider existing OT infrastructure and assess and develop an integration plan for AI systems. \n- Consider using test infrastructure before deployment to production systems, if possible (see Principle 3 – Establish AI Governance and Assurance Frameworks)." }, { "requirementID": "2.4.5", "requirementText": "Encourage push-based architectures where data is pushed out of the OT network for AI systems to use without persistent access into the OT network" }, { "requirementID": "2.4.6", "requirementText": "Prioritize the organization’s control over critical functions that AI systems may integrate with or enable when hosting AI systems locally or in the cloud.Ensure there are failsafe mechanisms that revert to traditional automation or manual for any AI-enabled system processes." }, { "requirementID": "2.4.7", "requirementText": "Integrate AI systems the same as any new OT systems: test AI systems for safety impacts (e.g., latency, interoperability) and verify they work within existing device management policy" }, { "requirementID": "3.1.5 - Continuously validate and verify the performance of AI systems", "requirementText": "Make sure they (AI systems) meet the organization’s objectives and regulatory requirements." }, { "requirementID": "3.2 Integrating AI into Existing Security and Cyber Security Frameworks", "requirementText": "When integrating AI into OT environments, critical infrastructure owners and operators should consider the existing security and cybersecurity frameworks that govern these systems and embed AI system assessments within existing risk evaluation, mitigation, and monitoring processes. This means that traditional cybersecurity requirements, vulnerability management, and critical infrastructure regulations must be factored in when integrating AI systems." }, { "requirementID": "3.4 Navigating Regulatory and Compliance Considerations for AI in OT", "requirementText": "Critical infrastructure owners and operators should evaluate the applicability of current AI technical standards in their OT domain as AI technical standards are rapidly evolving." }, { "requirementID": "4.2.2 - Design functional safety procedures that account for the AI system", "requirementText": "Each critical infrastructure sector has its own safety states and procedures. Per Principle 2 – Consider AI Use in the OT Domain, critical infrastructure owners and operators should review how they are integrating the AI system into their existing procedures and create new safe use and implementation procedures that focus on the AI system integration into the OT environment." } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "AIS-02", "requirementText": "Establish, document and maintain baseline requirements for securing applications." }, { "requirementID": "AIS-04", "requirementText": "Define and implement a secure software development lifecycle (SDLC) process for application requirements analysis, planning, design, development, testing, deployment, and operation in accordance with security requirements defined by the organization." }, { "requirementID": "AIS-11", "requirementText": "Establish security boundaries for agents." }, { "requirementID": "AIS-15", "requirementText": "Implement mechanisms enabling the model to clearly distinguish user-provided input instructions from data and system instructions (e.g., system prompts)." }, { "requirementID": "DSP-07", "requirementText": "Develop systems, products, and business practices based upon a principle of security by design and industry best practices." }, { "requirementID": "DSP-08", "requirementText": "Develop systems, products, and business practices based upon a principle of privacy by design and industry best practices. Ensure that systems' privacy settings are configured by default, according to all applicable laws and regulations." }, { "requirementID": "DSP-24", "requirementText": "Ensure training-data differentiation and relevance to the intended use of the AI Model." }, { "requirementID": "IAM-11", "requirementText": "Define, implement and evaluate processes and procedures for customers to participate, where applicable, in the granting of access for agreed, high risk (as defined by the organizational risk assessment) privileged access roles." }, { "requirementID": "IAM-14", "requirementText": "Define, implement and evaluate processes, procedures and technical measures for authenticating access to systems, application and data assets, including multifactor authentication for at least privileged user and sensitive data access. Adopt digital certificates or alternatives which achieve an equivalent level of security for system identities." }, { "requirementID": "IAM-17", "requirementText": "Define policy and procedure for \"need to know\" access to knowledge, information and data within the organization and in the context of the AI system to be applied when regulating access to resources." }, { "requirementID": "IAM-18", "requirementText": "When allowing model output modification of AI generated output, establish a role for this access and allow changes only by authorized identities." }, { "requirementID": "IAM-19", "requirementText": "Restrict agents' access to the tools and plugins necessary for the activity or use case at hand, ensuring adherence to the principles of need-to-know and least privilege." }, { "requirementID": "IPY-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for interoperability and portability including requirements for: a. Communications between application interfaces b. Information processing interoperability c. Application development portability d. Information/Data exchange, usage, portability, integrity, and persistence Review and update the policies and procedures at least annually or upon significant changes." }, { "requirementID": "MDS-07", "requirementText": "Define, implement, and evaluate processes, procedures, and technical measures for Model Hardening to mitigate relevant adversarial attacks as identified in the Threat Analysis and Adversarial Threat Analysis." }, { "requirementID": "MDS-13", "requirementText": "Adopt secure model formats and processes for AI model serialization where applicable." }, { "requirementID": "TVM-02", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures to protect against malware and malicious instructions. Review and update the policies and procedures at least annually or upon significant changes." } ] } }, "CoSAI": { "Establish Risks and Controls for the AI Supply Chain": { "link": "https://github.com/cosai-oasis/ws1-supply-chain/blob/main/risks-and-controls-for-the-ai-supply-chain-v1.md", "requirements": [ { "requirementID": "3.1.1 Data Poisoning: Threats and Mitigations in AI Supply Chains - Unauthorized Data", "requirementText": "Training on copyrighted materials or using user data without consent:\nComplete data provenance with source verification" }, { "requirementID": "3.2.3 Application - Insecure Function Calling", "requirementText": "Security weaknesses in the interface between language models and external function capabilities that can be exploited through control flow manipulation, allowing attackers to execute unauthorized actions when models are integrated with plugins, tools, or service APIs:\nRestrict model-accessible functions to read-only operations where possible. Apply granular least-privilege principles to all model function access permissions. Implement role-based access control (RBAC) frameworks to limit model interaction capabilities based on authentication context." } ] }, "Model Context Protocol (MCP) Security": { "link": "https://github.com/cosai-oasis/ws4-secure-design-agentic-systems/blob/main/model-context-protocol-security.md", "requirements": [ { "requirementID": "3.2.2 Secure Delegation and Access Control", "requirementText": "To mitigate against privilege escalation, MCP servers should operate with the minimum privileges necessary. OAuth provides a widely adopted framework for secure delegation, with extensions that support fine-grained scope control and secure token flows (see MCP Authorization).\n- Leverage existing identity providers to provide user authentication using standards such as OIDC\n- Register MCP server as clients with the IAM provider. If the registration cannot happen a priori, then use Dynamic Client Registration\n- Do not passthrough the OAuth tokenss provided by the user\n- Perform token exchange with the authorization server to provide full accountability (RFC8693)\n- Reduce scopes for least privilege, such as removing write scopes when only read access is required (SEP-835 adds native support to define scopes in 2025-11-25 MCP specification)\n- User short-liven tokens and support proof-of-possession (DPoP) to prevent replay attacks (RFC9449)\n- Fine grained authorizations, through Rich Authorization Requests (RFC9396), limit requests to specific resources or tool parameters\nAll endpoint services should implement robust access control models, such as role-based access control (RBAC) or attribute-based access control and evaluate against claims made by the identity provider, such as role membership, job title, or work location. Additionally, robust policy languages including Open Policy Agent (OPA), Cedar, or OpenFGA provide robust, flexible, and secure protections." } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "2.1.1", "requirementText": "The entity shall ensure secure design, development, and operation of AI/ML systems by establishing and maintaining comprehensive, enforceable policies and procedures." }, { "requirementID": "2.1.4", "requirementText": "The entity shall control and manage changes to AI/ML systems in a way that maintains system security." }, { "requirementID": "2.2.2", "requirementText": "The entity shall ensure AI/ML systems are securely and consistently configured in accordance with approved security baselines and configuration standards." }, { "requirementID": "2.2.5", "requirementText": "The entity shall ensure that AI/ML applications are securely developed, tested, and maintained throughout their lifecycle in accordance with established security practices." }, { "requirementID": "2.3.1", "requirementText": "The entity shall integrate cyber security into the design and development of AI/ML models from the earliest stages, to proactively mitigate security risks and avoid retrofitting controls after deployment." }, { "requirementID": "2.3.2", "requirementText": "The entity shall establish and maintain security measures to protect AI/ML models from potential threats, ensuring their integrity and availability throughout the model lifecycle." }, { "requirementID": "2.4.1", "requirementText": "The entity shall tailor cyber security measures for AI/ML systems according to their specific application domains, implementing context-appropriate safeguards." }, { "requirementID": "3.1.1 Cyber Security Policies & Procedures - 1", "requirementText": "The entity should develop, implement, and maintain a set of comprehensive policies and procedures that specifically address the unique cyber security considerations associated with AI/ML systems." }, { "requirementID": "3.2.5 Application Security - 1", "requirementText": "The entity should implement a secure software development lifecycle (SDLC) for AI/ML applications that includes security requirements definition, secure coding practices, security testing, and post-deployment security reviews." }, { "requirementID": "3.3.1 Security by Design for AI/ML Models - 1", "requirementText": "The entity should ensure that cyber security processes are integrated into AI/ML models from the earliest stages of their design and development." }, { "requirementID": "3.3.1 Security by Design for AI/ML Models - 3", "requirementText": "The entity should ensure that AI/ML models comply with established cyber security standards and best practices, including data minimization, least privilege, and separation of duties principles." }, { "requirementID": "3.3.2 AI/ML Model Security - 3", "requirementText": "The entity should implement safeguards to protect against adversarial attacks, such as adversarial training, robust input validation, and continuous monitoring for anomalous model behavior indicative of potential cyber security incidents." }, { "requirementID": "3.3.3 Inference Security - 1", "requirementText": "The entity should implement measures to secure the process of making predictions with AI/ML models, ensuring the integrity, confidentiality, and availability of inference data." }, { "requirementID": "3.3.3 Inference Security - 2", "requirementText": "The entity should adopt strategies to mitigate threats against inference security, such as model inversion attacks, membership inference attacks, and adversarial attacks." }, { "requirementID": "3.3.3 Inference Security - 3", "requirementText": "The entity should deploy defensive measures, including but not limited to robust input validation, to protect against inference attacks aiming to exploit or reverseengineer AI/ML models." }, { "requirementID": "3.4.1 Context-Specific AI/ML Security - 1", "requirementText": "The entity should identify and address the unique cyber security requirements of AI/ML systems based on their specific application context (e.g. medical, automotive, industrial, customer service, etc.)." }, { "requirementID": "3.4.1 Context-Specific AI/ML Security - 2", "requirementText": "For safety-critical applications (e.g. self-driving cars, medical diagnostics), the entity should prioritize robustness, reliability, and fail-safe mechanisms in AI/ML design and deployment." }, { "requirementID": "3.4.1 Context-Specific AI/ML Security - 3", "requirementText": "For privacy-sensitive applications (e.g. personal assistants, customer service bots), the entity should prioritize data protection, privacy, and transparency in AI/ML design and deployment." }, { "requirementID": "3.4.1 Context-Specific AI/ML Security - 4", "requirementText": "The entity should follow industry-specific guidelines, best practices, and regulations for AI/ML cyber security, as applicable to their application context." }, { "requirementID": "3.4.2 Fail-Safe & Backup for AI/ML Systems - 1", "requirementText": "The entity should implement robust fail-safe mechanisms for AI/ML systems to prevent failures and ensure system stability under unexpected conditions." }, { "requirementID": "3.5.2 Defending Against AI/ML Attacks - 2", "requirementText": "Security measures should be integrated during the development and deployment of AI/ML models to minimize the potential impact of attacks." } ] } }, "Databricks": { "The Databricks AI Security Framework": { "link": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirements": [ { "requirementID": "DASF 25: Use retrieval augmented generation (RAG) with large language models (LLMs)\n", "requirementText": "Generating relevant and accurate responses in large language models (LLMs) while avoiding hallucinations requires grounding them in domain-specific knowledge. Retrieval augmented generation (RAG) addresses this by breaking down extensive datasets into manageable segments (“chunks”) that are “vector embedded.” These vector embeddings are mathematical representations that help the model understand and quantify different data segments. As a result, LLMs produce responses that are contextually relevant and deeply rooted in the specific domain knowledge." }, { "requirementID": "DASF 41: Platform security — secure SDLC\n", "requirementText": "Databricks engineering integrates security throughout the software development lifecycle (SDLC), encompassing both technical and process-level controls under the oversight of our chief security officer (CSO). Activities within our SDLC include:\n- Code peer reviews\n- Static and dynamic scans for code and containers, including dependencies\n- Feature-level security reviews\n- Annual software engineering security training\n- Cross-organizational collaborations between security, product management, product security and security champions\n- Controls to prevent over-disclosure of technical information about AI systems and organizational details that could enable adversarial targeting\nThese development controls are augmented by internal and external penetration testing programs, with findings tracked for resolution and reported to our executive team. Databricks’ processes undergo an independent annual review, the results of which are published in our SOC 2 Type 2 report, available upon request." }, { "requirementID": "DASF 54: Implement AI guardrails", "requirementText": "AI Guardrails allow users to configure and enforce data compliance at the model serving endpoint level and to reduce harmful content on any requests sent to the underlying model. Bad requests and responses are blocked, and a default message is returned to the user. You can configure and enforce data compliance at the model serving endpoint level and reduce harmful content on any requests sent to the underlying model with AI Guardrails. With AI Guardrails, you can configure the following controls on your AI system:\n- Safety filtering prevents your model from interacting with unsafe and harmful content, like violent crime, self-harm, and hate speech.\n- Personally identifiable information (PII) detection to detect any sensitive information (such as names, addresses, and credit card numbers) for users\n- Topic moderation to list a set of allowed topics. Given a chat request, this guardrail flags the request if its topic is not one of the permitted topics.\n- Keyword filtering will specify different sets of invalid keywords for both the input and the output. One potential use case for keyword filtering is so the model does not talk about competitors.\n\n" }, { "requirementID": "DASF 64: Limit access from AI models and agents", "requirementText": "Allow AI models and agents access to enterprise resources based on the principle of least privilege. In Unity Catalog, a \"securable object\" is any object that can be permissioned to a principal (e.g., user, service principal, or group) and is organized hierarchically. Treat AI as a principal and assign permissions accordingly. \nWith on-behalf-of-user authentication, agents deployed via Mosaic AI model serving can access Databricks resources using the identity of the Databricks end user who queried the agent. This enables accessing sensitive information on a per-user basis, with fine-grained enforcement of data access control in Unity Catalog.\n\n" }, { "requirementID": "DASF 68: Use Securely Hosted Managed MCP Servers", "requirementText": "Model Context Protocol (MCP) servers act as bridges that let AI agents access external data and tools. Instead of building these connections from scratch, you can use securely hosted Databricks managed MCP servers to instantly connect your agents to data stored in Unity Catalog, vector search indexes, and custom functions.\n\n" }, { "requirementID": "DASF 73: Register prompts", "requirementText": "Although not quite as strong a control as a good safety filtering, prompt injection or jailbreak detection model, a good system prompt can sometimes be the difference between an attack being successful or not. MLflow prompt registry is a powerful tool that streamlines prompt engineering and management in your GenAI applications. It enables you to version, track, test and reuse prompts across your organization, helping maintain consistency and improving collaboration in effective prompt development." } ] } }, "ENISA": { "Multilayer Framework for Good Cybersecurity Practices for AI": { "link": "https://www.enisa.europa.eu/sites/default/files/publications/Multilayer%20Framework%20for%20Good%20Cybersecurity%20Practices%20for%20AI.pdf", "requirements": [ { "requirementID": "Poisoning", "requirementText": "Similarly, for poisoning attacks, processes that maintain the security levels of ML components over time should be implemented, the exposure level of the used model should be assessed, the training data set should be enlarged as much as possible to reduce its susceptibility to malicious samples, and pre-processing steps that clean the training data from such malicious samples must also be considered." }, { "requirementID": "Networking 6", "requirementText": "Have you defined/developed/used specific cyber measurements/metrics at the national level that AI stakeholders are required to use?" } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.1.2-1", "requirementText": "As part of deciding whether to create an AI system, a System Operator and/or Developer shall conduct a thorough assessment that includes determining and documenting the business requirements and/or problem they are seeking to address, along with associated AI security risks and mitigation strategies." }, { "requirementID": "Provision 5.1.2-1.1", "requirementText": "Where the Data Custodian is part of a Developer's organization, they shall be included in internal discussions when determining the requirements and data needs of an AI system." }, { "requirementID": "Provision 5.1.2-2", "requirementText": "Developers and System Operators shall ensure that AI systems are designed and implemented to withstand adversarial AI attacks, unexpected inputs and AI system failure." }, { "requirementID": "Provision 5.1.2-3", "requirementText": "To support the process of preparing data, security auditing and incident response for an AI system, Developers shall document and create an audit trail in relation to the AI system. This shall include the operation, and lifecycle management of models, datasets and prompts incorporated into the system." }, { "requirementID": "Provision 5.1.2-4", "requirementText": "If a Developer or System Operator uses an external component, they shall conduct an AI security risk assessment and due diligence process in line with their existing software development processes, that assesses AI specific risks." }, { "requirementID": "Provision 5.1.2-5", "requirementText": "Data Custodians shall ensure that the intended usage of the system is appropriate to the sensitivity of the data it was trained on as well as the controls intended to ensure the security of the data." }, { "requirementID": "Provision 5.1.2-5.1", "requirementText": "Organizations should ensure that employees are encouraged to proactively report and identify any potential security risks in AI systems and ensure appropriate safeguards are in place." }, { "requirementID": "Provision 5.1.2-6", "requirementText": "Where the AI system will be interacting with other systems or data sources, (be they internal or external), Developers and System Operators shall ensure that the permissions granted to the AI system on other systems are only provided as required for functionality and are risk assessed." }, { "requirementID": "Provision 5.1.2-7", "requirementText": "If a Developer or System Operator chooses to work with an external provider, they shall undertake a due diligence assessment and should ensure that the provider is adhering to the present document." } ] }, "SAI 002 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_gr/SAI/001_099/002/01.01.01_60/gr_SAI002v010101p.pdf", "requirements": [ { "requirementID": "6.1.2 Cybersecurity hygiene - 4", "requirementText": "Strong access controls should be in place, applying the principle of least privilege. These stand alongside limits to the number of queries allowed to be made against a model in a period of time." }, { "requirementID": "6.1.3 Supply chain security - 5", "requirementText": "Using additional security (for example cryptographic protection of data) to protect the most critical functions." }, { "requirementID": "6.5 - Following standard cybersecurity good practice", "requirementText": "Including following the principle of least privilege when accessing data." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.1.2-1", "requirementText": "\"As part of deciding whether to create an AI system, a System Operator and/or Developer shall conduct a thorough assessment that includes determining and documenting the business requirements and/or problem they are seeking to address, along with potential AI security risks and mitigation strategies.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout assessing whether an AI system is required to meet the business requirements, systems can be unnecessary or poorly suited for their environment, leading to lack of compliance, unnecessary complexity, increased attack surface, unexpected behaviour, and security vulnerabilities.\n\nExample Measures/Controls 1:\nConduct Business Alignment Review: Use ISO/IEC 25059 [i.16] as a quality model to review and document business requirements for the AI system to ensure that design choices align with the organization's needs and objectives.\n\nExample Measures/Controls 2:\nPerform Risk Assessment: Conduct and document an AI-specific risk assessment covering data classifications, logging risks of personal data and their mitigations in DPIAs. Cover expected data volume, types of integration, the model's complexity, architecture, and number of parameters. For more information on these risk factors see the NCSC Principles for Machine Learning [i.5] and NCSC Guidelines for Secure AI system development [i.6].\n\nExample Measures/Controls 3:\nIntegrate Risk Management and Governance Frameworks: Embed AI system assessments within both a formal Risk Management Framework (RMF) and the AI governance structure to ensure thorough risk evaluation, consistent mitigation, and monitoring before key organizational decisions." }, { "requirementID": "Provision 5.1.2-1.1", "requirementText": "\"Where the Data Custodian is part of a Developers organization, they shall be included in internal discussions when determining the requirements and data needs of an AI system.\" (ETSI TS 104 223 [i.1])\nRelated threats/risks:\nFailure to include the Data Custodian in discussions about AI system requirements and data needs can result in noncompliance with data governance policies, inappropriate data usage, or insufficient safeguards for sensitive data, increasing the risk of data breaches or regulatory violations.\nExample Measures/Controls:\nEnsure collaboration with the Data Custodian: During the design and development phases to define data requirements to identify regulatory compliance requirements. Ensure that Data Custodians are able to balance additional risks to data that come from the AI system with intended mitigations and the business need. " }, { "requirementID": "Provision 5.1.2-2", "requirementText": "\"Developers and System Operators shall ensure that AI systems are designed and implemented to withstand adversarial AI attacks, unexpected inputs and AI system failure.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nOrganizations are not always successful in preventing breaches and so defence in depth requires planning for and responding to compromise. The absence of defence in depth can lead to infringement of applicable regulation.\n\nExample Measures/Controls:\nApply Secure by Design Principles: Integrate security into the AI system's design phase by conducting threat modelling. Threat modelling covers both traditional cyber threats and AI-specific ones that might be introduced by the design choices. Incorporate standardized security controls in the system design controls to mitigate risks. Document each standardized control used in the design phase, and ensure it is integrated with specific test cases to verify its effectiveness during system testing. Ensure monitoring controls as well as incident response and recovery from failures are addressed in threat mitigation and they are documented in the system's design." }, { "requirementID": "Provision 5.1.2-3", "requirementText": "\"To support the process of preparing data, security auditing and incident response for an AI system, Developers shall document and create an audit trail in relation to the AI system. This shall include the operation, and life cycle management of models, datasets and prompts incorporated into the system.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nA lack of audit trails can lead to untraceable changes or unauthorized adjustments, complicating incident response, forensic investigations, and regulatory compliance.\n\nExample Measures/Controls:\nAutomated Audit Trails for ML Operations (MLOps) and System changes: Implement automated logging for all critical operations related to model training, dataset changes, prompts and parameter adjustments. For critical systems with compliance requirements, use WORM (write-once, read-many) storage to store logs, ensuring they remain tamperproof and accessible for audits." }, { "requirementID": "Provision 5.1.2-4", "requirementText": "\"If a Developer or System Operator uses an external component they shall conduct an AI security risk assessment and due diligence process in line with their existing software development processes, that assesses AI specific risks.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nThird-party components introduce risks through possible vulnerabilities in the external vendor's security practices which might not be to the same standard. This includes operating systems and libraries, container images, programming packages as well as models and datasets. Large models contain general purpose functionality, ensure that specific risks are mitigated or otherwise managed.\nIn the context of product safety regulation, risks to AI security include risks that can lead to hazardous situations. Hazardous situations can include risks to fundamental rights, health and safety.\nFor further information about risk management in the context of the EU's AI Act, see JTC21024 [i.146].\n\nExample Measures/Controls:\nSecurity Due-Diligence for External Components: Mandate a risk assessment process before a component (including external models) can be used, covering provenance, known risks, and when personal data is used a DPIA. Safeguard provenance by mandating in internal standards that components can only be sourced by trusted and approved sources, documenting source, version, licencing, history, and other related artifacts (e.g. Model Card for models); use checksums to verify integrity." }, { "requirementID": "Provision 5.1.2-5", "requirementText": "\"Data Custodians shall ensure that the intended usage of the system is appropriate to the sensitivity of the data it was trained on as well as the controls intended to ensure the security of the data.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nMisalignment between the intended usage of the AI system and the sensitivity of the data it was trained on can result in inappropriate data exposure, inadequate security controls, and regulatory non-compliance, leading to potential data breaches and misuse of personal data or other confidential information.\n\nExample Measures/Controls:\nEnsure Data Custodian Assurance: Require Data Custodians to review system's intended usage and the data security controls to ensure compliance and balancing these risks with business needs." }, { "requirementID": "Provision 5.1.2-5.1", "requirementText": "\"Organizations should ensure that employees are encouraged to proactively report and identify any potential security risks in AI systems and ensure appropriate safeguards are in place.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nA lack of proactive reporting and identification of security risks in AI systems can lead to undetected vulnerabilities, increasing the likelihood of security breaches, data leaks, or misuse of AI, with potentially significant operational, financial, and reputational consequences.\n\nExample Measures/Controls:\nSupport proactive reporting of security risks. Establish a clear, accessible process for employees to report potential security risks in AI systems, encourage a culture of proactive risk identification by providing training, communication channels, transparent handling, and recognition for reporting issues." }, { "requirementID": "Provision 5.1.2-6", "requirementText": "\"Where the AI system will be interacting with other systems or data sources, (be they internal or external), Developers and System Operators shall ensure that the permissions granted to the AI system on other systems are only provided as required for functionality and are risk assessed.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nThere is huge potential and interest in \"agentic systems\", where an AI system can decide and conduct its own actions, typically through integrations with other systems. The actions taken by an AI system are not fully predictable, and can be coerced by an attacker. This risk introduces the potential for unauthorized access, data exfiltration, and privilege escalation.\n\nExample Measures/Controls:\nLeast-privilege access to data and systems accessed by AI System. Mandate a risk assessment process before a component can be used covering provenance, known risks, and evaluations. Ensure that the assessment covers all possible model states, not just the designed or expected ones." }, { "requirementID": "Provision 5.1.2-7", "requirementText": "\"If a Developer or System Operator chooses to work with an external provider, they shall undertake a due diligence assessment and should ensure that the provider is adhering to the present document.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nCollaborating with external providers without assessing their adherence to ETSI TS 104 223 [i.1] can lead to increased vulnerabilities, lack of regulatory compliance, insecure systems, or inadequate response protocols, which could compromise the entire system's security.\n\nExample Measures/Controls:\nSecurity Review of External Providers: Verify the external provider's implementation of ETSI TS 104 223 [i.1] with the external provider and their overall regulatory compliance." } ] }, "TR 104 222 - Securing Artificial Intelligence; Mitigation Strategy Report": { "link": "https://www.etsi.org/deliver/etsi_tr/104200_104299/104222/01.02.01_60/tr_104222v010201p.pdf", "requirements": [ { "requirementID": "5.2.2 - Enhance data quality - 2", "requirementText": "A general method in pre-processing data as a means to minimize risks of data manipulations was proposed in [i.15]. It tries to estimate the quality of candidate training data and only uses data of sufficient quality for training the model. The authors consider the use-case of threat intelligence systems and use semi-supervised learning to judge the trustworthiness of data. The limitation of this approach consists in that it mostly considers increasing data quality with respect to benign problems (such as unintentional wrong labelling) and does not address targeted and optimized attacks. " }, { "requirementID": "5.2.2 - Data sanitisation - 2", "requirementText": "TRIM [i.12] iteratively estimates the model parameters and trains on the subset of best-fitting input points at the same time, until convergence is reached. Its results are much better than those of RONI. However, TRIM is devised for linear regression only and thus not applicable to (deep) neural networks." }, { "requirementID": "5.2.2 - Data sanitisation - 3", "requirementText": "Provenance-based [i.13]: another extension of RONI uses (presumably correct) meta data about data provenance and first clusters data accordingly. Due to the additional information used, this approach achieves better results than RONI and is more efficient by a certain factor (essentially, the average cluster size), since the model does not need to be retrained for each individual point but only the cluster centroids. The intuition of [i.13] is that for data points of common provenance the probability of being poisoned is strongly correlated. Applying this technique assumes the existence and correctness of information on data provenance." }, { "requirementID": "5.2.2 - Data sanitisation - 4", "requirementText": "Keyed Non-parametric Hypothesis Tests (KNHT) [i.17] assumes a set of clean training data which describe intended data distribution and inspect newly-collected ones. The rationale is to compare the two set of data and if their similarity is insufficient, the newly collected ones are further inspected. KNHT does the data distribution comparison after mapping the two data distributions into another space via a set of functions with secret keys." }, { "requirementID": "5.2.2 - Blocking poisoning", "requirementText": "Gradient shaping [i.14] starts from the new insight that poisoning (and backdoor) attacks exhibit larger gradients with differing orientations for poisoned data as compared to clean data. Hence, gradient shaping aims to prevent these properties in the gradients from occurring during model training (e.g. by cropping large values). The gradient shaping technique does not use additional information about the training dataset, and its run-time overhead is negligible. The results presented in [i.14] look quite promising for several attacks, but like other approaches, the strategy cannot thwart strong state-of-the-art attacks." }, { "requirementID": "5.3.2 Model enhancement mitigations against backdoor attacks - 1", "requirementText": "Spectral signature [i.18]: it is observed that spectral signatures can be used to identify and remove poisoned inputs, where spectral signatures are the spectrum of the covariance of a feature representation learned by the neural network." }, { "requirementID": "5.3.2 Model enhancement mitigations against backdoor attacks - 2", "requirementText": "Activation clustering [i.19]: triggers can be detected from the neuron activation in the final hidden layer of a network. It is observed that input with triggers have different patterns of neuron activation from benign input. Hence, by using unsupervised clustering on neuron activation in the final hidden layer, input with triggers can be identified. The addressed model can then be retrained excluding triggers." }, { "requirementID": "5.3.2 Model enhancement mitigations against backdoor attacks - 3", "requirementText": "Retraining [i.20]: retraining the model with a small subset of clean training data is explored for correcting backdoor-embedded neural networks." }, { "requirementID": "5.3.2 Model enhancement mitigations against backdoor attacks - 4", "requirementText": "Fine-pruning [i.22]: pruning a neural network eliminates some less important neurons [i.21]. It is a means of raising the success bar of backdoor attacks at some cost of accuracy. Pruning technique is effective when attackers are unaware of pruning protection. Fine-pruning retrains the model after pruning to address pruningaware backdoor attacks. By applying fine-pruning, a backdoor-embedded neural network can be restored." }, { "requirementID": "5.3.2 Model enhancement mitigations against backdoor attacks - 5", "requirementText": "Neural-Cleanse [i.23]: small input perturbations are reverse engineered from the model. Those perturbations trigger backdoor behavior in the model and then a backdoored class can be identified. The addressed model can then be patched by retraining with purified data. This process does backdoor un-learning. Neural cleanse method relies on an assumption that the trigger for the backdoor-infected class is smaller than the median size of the reverse engineered trigger for all classes. This assumption fails when triggers have various sizes." }, { "requirementID": "5.3.2 Model enhancement mitigations against backdoor attacks - 6", "requirementText": "TABOR [i.24]: the backdoor detection task is transformed to a non-convex optimization problem with a newly-designed objective function to narrow down search space. Hence, input with triggers can be efficiently reverse-engineered. After triggers are found, by retraining the model with purified data, the model can be restored." }, { "requirementID": "5.3.3 Model-agnostic mitigations against backdoor attacks - 1", "requirementText": "STRIP [i.25]: a runtime trigger detection scheme is designed based on an assumption that an input with trigger is insensitive to perturbations. By applying a set of perturbations, a given inference input is then represented as a set of data samples. The classification entropy introduced by the set of data samples is a quantitative measure on how likely the given inference input contains a trigger. Inputs with low classification entropy even when applied strong perturbations are unlikely to be benign. The assumption is not universal. But this method does not require to know model parameters and can be performed at run-time." }, { "requirementID": "5.3.3 Model-agnostic mitigations against backdoor attacks - 2", "requirementText": "SUTO-NEO [i.27]: it is a trigger detection and prediction restoration scheme for image classification task. With an assumption that only one trigger exists and the trigger location is fixed on the image, an input image can be analyzed to see if a trigger exists. Given an input image, the potential trigger location can be searched by comparing classification results of the intact image and the image with a dominant color block at a random position. If the results diverse, the position likely has a trigger. To further confirm about it, the image block covered by the color block is implanted to other known-results training images. If those known-result training images with the patched image block have different prediction results from before, the position and the trigger are confirmed. To restore the classification result, the image with a trigger is patched with a block of the dominate color and then the corresponding predication can be restored." }, { "requirementID": "5.3.3 Model-agnostic mitigations against backdoor attacks - 3", "requirementText": "SentiNet [i.26]: by combining techniques of model interpretability and object detection, SentiNet is a mean of trigger detection at run-time against localized and universal backdoor attacks on image classifiers. It can detect more than one trigger with various sizes." }, { "requirementID": "5.3.3 Model-agnostic mitigations against backdoor attacks - 4", "requirementText": "Februus [i.28]: researchers proposed a trigger deactivation method by input restoration technique against input-agnostic triggers. They analyzed which region of the input has the most impact on the classification result via visual explanation tool GradCAM [i.29] and then deactivate potential triggers. The most influential region is replaced by a neutralized color box and then the masked input is restored by generative adversarial networks. This method needs access to not only the model but also the training dataset." }, { "requirementID": "5.3.3 Model-agnostic mitigations against backdoor attacks - 5", "requirementText": "Trigger reverse engineering: as described earlier, neural cleanse method [i.23] and TABOR [i.24] both consist of two steps. The first step is to reverse engineer triggers from the addressed model and the second step is to retrain the model by purified data Although at deployment stage, the retraining step cannot be performed, the knowledge of triggers from the first step helps on detecting backdoored models." }, { "requirementID": "5.3.3 Model-agnostic mitigations against backdoor attacks - 6", "requirementText": "Meta Neural Trojan Detection (MNTD) [i.30]: a meta neural analysis framework is proposed for classifying benign models and backdoored models. A meta classifier can be built to classify models by representing a model as a sequence of predictions from fine-tuned queries. This method needs to build shadow models from clean datasets and self-generated backdoored datasets. But the method only requires query access to the addressed model with zero knowledge of model parameters." }, { "requirementID": "5.3.3 Model-agnostic mitigations against backdoor attacks - 7", "requirementText": "DeepInspect [i.31]: it detects backdoor without knowledge of model parameters and clean or training datasets. The method consists of three steps: model inversion to get a surrogate training dataset, trigger generation by conditional generative adversarial networks, and anomaly detection based on statistical hypothesis testing. DeepInspect detects backdoor in models with less prior knowledge at a cost of slightly lower detection rate." }, { "requirementID": "5.3.3 Model-agnostic mitigations against backdoor attacks - 8", "requirementText": "Universal Litmus Patterns (ULPs) [i.32]: similar to MNTD, Universal Litmus Patterns fine-tunes a set of optimized input images for determining whether a model contains backdoors. Although ULPs cannot deactivate triggers or backdoors, it provides an efficient and effective backdoor detection technique against single trigger-based backdoor attacks." }, { "requirementID": "6.2.1 - Transferability", "requirementText": "Transferability is a feature of adversarial examples. Some adversarial examples crafted for one model can fool a different model with a high probability. They are called transferable adversarial examples. The work in [i.33] demonstrates the transferability of adversarial examples among several machine learning models. A later work in [i.34] generates adversarial examples with better transferability.\n\nThe area of mitigations against evasion attacks has been rapidly evolving with new mitigations being proposed and broken at high speed. On the one hand, most mitigations empirically target very specific attack types, but are easily broken once the attacker can adapt to known mitigations, as has been repeatedly demonstrated [i.35], [i.36] and [i.37]. Such mitigations, even if they can be broken by adaptive attacks, can still be sufficient for certain use cases, since adaptive attacks take more efforts on the part of the attacker and are hence less likely to occur in practice. On the other hand, some mitigations are derived from theoretic analysis and formal reasoning in a certain threat model and hence remain effective against a wide range of attacks, including certain adaptive attacks. However, if the threat model does not capture the practical constraints of attacks sufficiently well, attackers can circumvent the threat model easily, and in this case these mitigations can be just as vulnerable as empirical ones. It is, for instance, well-known that the perturbation metrics used in threat models do not reflect human perception very well. In addition, there is a trade-off between the robustness and invariance to perturbations a model can have, which can likewise be exploited [i.38]." }, { "requirementID": "6.2.2 - 1", "requirementText": "Transformation: to narrow down the attack surface of manipulating inputs, some data transformation techniques are applied. Adversarial perturbations are expected to be diminished through data transformation. For images, researchers took data compression as defense, for example, using Joint Photographic Experts Group (JPEG) for input data transformation [i.39]. Later on more data transformation techniques are proposed [i.40] as defense, including low-pass filtering, JPEG compression, Principal Components Analysis (PCA), soft-thresholding, patchwise PCA, low resolution wavelet approximation, against gradient-based attacks and concluded that JPEG compression tends to outperform others in most experimental cases. However, it has been [i.41] shown how to generate adversarial examples which survive JPEG compression. The compression defense [i.42] is advanced by combining JPEG compression and random compression level and showed that the new technique is effective against Carlini-Wagner's (CW) attack and DeepFool attack. Researchers [i.43] also studied data transformation as defense, including cropping-rescaling, bit-depth reduction, JPEG compression, total variance minimization, and image quilting on ImageNet and show that total variance minimization and image quilting are effective." }, { "requirementID": "6.2.2 - 2", "requirementText": "NULL labelling [i.44]: for classifiers, instead of denoising adversarial inputs, NULL labelling technique lets the addressed model learn how likely an input is adversarial by adding a NULL label to the label space. As a result, adversarial examples are classified as NULL with a high probability while the model accuracy remains." }, { "requirementID": "6.2.2 - 3", "requirementText": "Adversarial training is one of the most prominent techniques explored for mitigating evasion attacks. Adversarial training consists in training the model on a training data set augmented with correctly labelled adversarial examples, thus making sure the model will not be fooled by adversarial examples. The adversarial examples used for training can be computed using a variety of methods for mounting evasion attacks, such as the Fast Gradient Sign Method (FGSM) [i.45]. Some existing work [i.46] has shown that using the Projected Gradient Descent (PGD) method creates models with quite good robustness against evasion attacks, including adaptive attacks. This is probably due to the fact that PGD can be seen as a generalization of other attack methods, and can hence find the strongest adversarial examples for a given threat model with high probability. The comparatively good effectiveness of adversarial training using PGD compared to other defense methods is also confirmed by [i.47]. \nStill, adversarial training only guarantees a certain level of robustness against evasion attacks, which is not sufficient for some applications. Adversarial training also increases the run-time for training by a certain factor that depends on the level of robustness provided." }, { "requirementID": "6.2.2 - 4", "requirementText": "Regularization builds on the idea that preventing small input perturbations from changing model output can defend against adversarial examples. In mathematical terms this translates to bounding the Lipschitz constant [i.48] of the function implemented by the neural network or its gradient [i.49] by adapting the training procedure. In fact, there is a strong relation between these two approaches, since every differentiable function with bounded gradient is Lipschitz continuous (although the implied constant can be large, and the function in question is not everywhere differentiable). \nRegularization can certifiably improve robustness, but faces several limitations [i.50]. In many cases, the bounds on the Lipschitz constant or gradient, respectively, are not strictly enforced, but only encouraged by penalty terms. If the bounds are indeed enforced, this impacts the generalization performance of the network and thus its overall accuracy on complex tasks." }, { "requirementID": "6.2.2 - 5", "requirementText": "Certifiable training: in the research line of robustness verification, certifiable training techniques attempt to train neural networks to improve the lower bound of robustness. Existing methods include designing specific regularizations, such as Lipschitz regularization, and randomized smoothing. Randomized smoothing technique [i.51] is derived from probabilistic robustness verification. For classifiers, a model is firstly trained by the training dataset augmented with Gaussian noises, where the output is a probability distribution over all classes. The final model is trained via the augmented dataset with labels indicating the class with the highest probability. Via randomized smoothing techniques, the derived model has probabilistic verifiable robustness." }, { "requirementID": "6.2.2 - 6", "requirementText": "Gradient masking: to defend against gradient-based attacks, gradient masking is one approach. For example, shattered gradients introduce non-differentiable operations such that attackers cannot get correct gradients for crafting adversarial examples. Another example is stochastic gradients which introduce randomness on gradients. However, the results of [i.35] and [i.36] pointed out that gradient masking approach can be overcame effectively. To recover the gradient signals from shattered gradients, backward pass differentiable approximation techniques are proposed to approximate gradients [i.36]. To mitigate the uncertainty of stochastic gradients, expectation over transformation techniques are proposed to estimate gradients [i.35]. Hence, gradient masking approach is much less effective against gradient-based attacks." }, { "requirementID": "6.2.2 - 7", "requirementText": "Distillation network [i.52]: it is also called defensive distillation. Distillation network is proposed as robust training technique for classifiers against adversarial examples. To train the distilled network, a teacher model is firstly trained on a training dataset. The distilled network is then trained on the same input and the corresponding output probabilities of the teacher network. The output probabilities are also called soft labels while the original labels of the training dataset are hard labels. Distillation network has been shown less effective against adaptive attackers [i.53]." }, { "requirementID": "6.2.2 - 9", "requirementText": "Model verification for certifiable robustness: Certifiable robustness is a conceptual theoretic description capturing model robustness against evasion attacks. An adversarial example is a crafted input close enough to an intact input where they are somewhat indistinguishable to human, yet the addressed model gives them different output. Figure 5 gives a conceptual illustration, where x is an input, x' is an adversarial example, y and y' are model output for x and x' respectively. The distance between x and x' is evaluated by certain metrics, such as Lp distance. The addressed model is the function mapping input space to output space. Certifiable robustness attempts to certify a lower bound of the perturbation range δ where no adversarial example exists. Existing examples include certifiable robustness for CNN [i.54], RNN [i.55] and GNN [i.56]." }, { "requirementID": "6.2.3 - 1", "requirementText": "Input transformation: for image classifiers, image transformations, such as rotation and shifting, are proposed to detect adversarial examples by constructing a detector from the training dataset with image transformations [i.69]. Experimental results show the effectiveness by the detection rate on MNIST and CIFAR-10 against FGSM and CW evasion attacks. Another example of image transformation is feature squeezing [i.70], such as reducing the color bit depth of each pixel and spatial smoothing. One more example is adding random perturbation. An adversarial example is close to an input within a certain bounded range, but the addressed model gives different outputs. When a given input is indeed an adversarial example, adding some perturbation on the given input can push the output back to the original class with a non-negligible probability. Based on this rationale, researchers [i.71] proposed adding random perturbations on input and analysing the resulting output to detect adversarial examples. Their experimental results show the effectiveness on MNIST, CIFAR10 and ImageNet against BIM, DeepFool and CW evasion attacks at the cost of generating a detector from the original training dataset." }, { "requirementID": "6.2.3 - 2", "requirementText": "Statistics: to detect adversarial examples, some research seek for proper statistics. For example, for convolution neural networks, binary classifiers are constructed from the statistics of convolutional layer results from normal training dataset and adversarial examples [i.72]. Experimental results show the effectiveness on ILSVRC2012 (ImageNet-based) dataset with AlexNet and VGGNet network architectures against L-BFGS attacks at the cost of constructing a binary classifier. This method needs knowledge of training dataset. Another example of statistics method uses the expected perturbed log-odds over random perturbations to test if a given input x classified as class y is manipulated while the true class is z [i.73]. This method needs to access the results before the final layer at runtime. Experimental results show the effectiveness on CIFAR-10 and ImageNet against PGD attacks at the runtime cost of operating multiple inference for one input." }, { "requirementID": "6.2.3 - 4", "requirementText": "Quantization methods [i.76] are proposed to diminish adversarial perturbations from input at inference time. The challenge is to find a proper quantization level such that perturbations are diminished while model accuracy remains. Trainable quantization method finds the suitable quantization level during training phase and applies it during inference phase. Experimental results show the effectiveness on MNIST and CIFAR-10 against FGSM, CW and JSMA attacks." }, { "requirementID": "6.2.3 - 5", "requirementText": "Ensemble: for classifiers, ensemble methods are proposed to mitigate adversarial perturbations. The assumption is that some adversarial examples misleading one classifier do not mislead other classifiers. Ensemble methods construct a set of classifiers to classify a new data input by a combination of their predictions. Researchers [i.77] proposed four ensemble constructions for model robustness, including random initial model parameters, similar but different network architectures, bagging, and adding Gaussian noise on training dataset. Their experimental results on MNIST and CIFAR-10 show that ensemble methods are effective against FGSM and BIM attacks. Another ensemble construction [i.78] is proposed by using different loss functions in the set of classifiers. Experimental results show that the ensemble method has better robustness against FGSM-based attacks at the cost of model accuracy on clean dataset. Some adversarial examples [i.33] can transfer among models, which are thus immune to ensemble methods." }, { "requirementID": "6.3.3 - 2", "requirementText": "Boundary differential privacy (ϵ-BDP) [i.83]: this method protects against model extraction attacks that use fine-tuned queries with differential property to infer the decision boundary of the addressed model. The proposed solution is to obfuscate the prediction responses near the decision boundary. Based on a perturbation algorithm called boundary randomized response, the ϵ -BDP method prevents adversaries from learning the decision boundary by a predefined precision regardless the amount of queries issued to the model prediction API. The technique is effective for both linear and non-linear models." }, { "requirementID": "6.3.3 - 4", "requirementText": "Rounding confidences [i.85]: with the aim to limit the information provided by model query APIs, this technique proposes to round confidence scores of the target model to some fixed precision. For decision trees, rounding confidence scores increase the chance of node identifier collisions and thus reduces attacker's success rate. The defense has also shown its effectiveness against extraction attacks in decision trees, but not in some other models." }, { "requirementID": "6.3.3 - 5", "requirementText": "Ensemble [i.85]: ensemble methods such as aggregation of predictions by a number of individual models, can be used to defend against model reverse engineering attack. In this research paper, the author found that ensemble methods' resilience to extraction attacks is higher, since attackers will only be able to obtain relatively coarse approximations of the target function." }, { "requirementID": "6.3.3 - 6", "requirementText": "Fingerprinting based on conferrable adversarial examples [i.87]: this defense addresses DNN-based classifiers and it extracts a set of inputs from the source model so that only surrogate models agree with the source model on the classification of such inputs, whereas (benign) reference models relative to the source model do not. These inputs called conferrable adversarial examples are a subclass of transferable adversarial examples that exclusively transfer with a target label from a source model to its surrogates. While this fingerprinting scheme is robust to almost all derivation and extraction attacks, strong adaptive attacks can still remove the fingerprint. To implement this protection, the defender has the full knowledge of the source model parameters, query access to the target model deployed by the attacker and a limited number of queries to verify whether the target model is a surrogate." }, { "requirementID": "6.3.3 - 7", "requirementText": "Fingerprinting the classification boundary [i.88]: this technique uses gradient descent method to find fingerprinting data points near the classification boundary of the addressed DNN-based classifier. These data points together with their predicted labels are used as the fingerprint for the addressed classifier. The model owner can identify that a suspect classifier is potentially pirated from the addressed model by querying its prediction API to get the labels of the fingerprinting data points. If most of the suspect classifier's predictions match those of the fingerprint for the owner's model, then the IP is compromised with a high probability. Authors claim that unlike watermarking, the technique does not incur in any accuracy loss for the addressed classifier, since it does not tamper its training or fine-tuning process at all." }, { "requirementID": "6.4.2 - 2", "requirementText": "Adversarial learning with privacy: to provide data privacy, one approach is to train a data representation from the original training dataset for the model training at a later time. The trained data representation reveals less sensitive information of training data set while keeping model accuracy as much as possible. One example [i.90] is addressing medical records in text format. Medical records are converted to another representation with less identity information. The process is also called de-identification. The de-identification process is composed by three training stages, including training an initial representation, training a potential adversary, retraining the representation against such adversary. By using de-identified data representation, given two data representations, an adversary cannot distinguish if they are converted from the same input data. Another example [i.91] is addressing facial images. The representation generation of training data set is trained with two objectives of optimizing model accuracy and maximizing adversarial reconstruction loss. The adversarial reconstruction loss is defined by predictions of a discriminator and a perceptual distance. Hence, the obtained data representation of the training dataset reveals less information while keeping model accuracy." }, { "requirementID": "6.4.2 - 3", "requirementText": "DP-SGD [i.93]: differential privacy [i.92] is a framework for quantitatively measuring the privacy guarantees provided by an algorithm. The basic idea is to add randomness to algorithm's behaviours. Learning with differential privacy provides provable privacy guarantees and mitigating the risks of exposing sensitive information about training data. For a model trained with differential privacy, the behaviours of the trained model is less affected by any single training data. Hence, it is difficult to tell which data record belongs to the training dataset by observing the model's behaviour. By using the framework of differential privacy, Differentially Private Stochastic Gradient Descent (DP-SGD) technique is proposed to train a model with provable membership privacy. DP-SGD adds noises to the gradients used in Stochastic Gradient Descent (SGD), which is the core of almost all deep learning algorithms. The parameters of the trained model are iteratively updated using random noises and gradients. The privacy comes from the added noises. However, DP-SGD imposes a significant classification accuracy loss for protecting large models on high dimensional data." }, { "requirementID": "6.4.2 - 4", "requirementText": "Adversarial regularization [i.94] mitigates membership inference attacks through adversarial training. The model's predictions on the members of its training dataset are indistinguishable from its predictions on the nonmembers of its training dataset from the same distribution. This requirement is captured as a min-max game optimization problem. The adversarial training algorithm is designed to train a model that minimizes both the prediction error and the maximum gain of the membership inference attack against it." }, { "requirementID": "6.4.3 - 1", "requirementText": "MemGuard [i.95] add noises to confidence scores to mitigate membership inference attacks. Assume attackers use a classifier to distinguish whether an input is in the training dataset. MemGuard crafts noises such that the manipulated confidence scores become adversarial examples to attacker's classifier. As a result, the attacker cannot successfully launch membership inference attack." }, { "requirementID": "6.4.3 - 2", "requirementText": "Prediction purification [i.96] purifies the confidence score vectors predicted by the target classifier by reducing their dispersion. As a result, attackers get less information from purified confidence score vectors. Prediction purifications can be built to mitigate only membership inference attack, only model inversion attack and both attacks. Moreover, from experimental results, prediction purification against membership inference attack is effective against model inversion attack and vice versa. The generation of purifiers needs a subset of training dataset. The generated purifier is composed with the addressed classifier model to purify the confidence score vectors. Experimental results show prediction purification can reduce the membership inference accuracy and increase the model inversion error at the cost of little classification accuracy drop and little distortion to the confidence scores." } ] } }, "EU ": { "EU AI Act": { "link": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirements": [ { "requirementID": "10.1 Data and Data Governance", "requirementText": "High-risk AI systems which make use of techniques involving the training of AI models with data shall be developed on the basis of training, validation and testing data sets that meet the quality criteria referred to in paragraphs 2 to 5 whenever such data sets are used." }, { "requirementID": "10.2 Data and Data Governance", "requirementText": "Training, validation and testing data sets shall be subject to data governance and management practices appropriate for the intended purpose of the high-risk AI system. Those practices shall concern in particular:\n(a) the relevant design choices;\n(b) data collection processes and the origin of data, and in the case of personal data, the original purpose of the data collection;\n(c) relevant data-preparation processing operations, such as annotation, labelling, cleaning, updating, enrichment and aggregation;\n(d) the formulation of assumptions, in particular with respect to the information that the data are supposed to measure and represent;\n(e) an assessment of the availability, quantity and suitability of the data sets that are needed;\n(f) examination in view of possible biases that are likely to affect the health and safety of persons, have a negative impact on fundamental rights or lead to discrimination prohibited under Union law, especially where data outputs influence inputs for future operations;\n(g) appropriate measures to detect, prevent and mitigate possible biases identified according to point (f);\n(h) the identification of relevant data gaps or shortcomings that prevent compliance with this Regulation, and how those gaps and shortcomings can be addressed." }, { "requirementID": "14.3 Human Oversight", "requirementText": "The oversight measures shall be commensurate with the risks, level of autonomy and context of use of the high-risk AI system, and shall be ensured through either one or both of the following types of measures:\n\n(a) measures identified and built, when technically feasible, into the high-risk AI system by the provider before it is placed on the market or put into service;\n\n(b) measures identified by the provider before placing the high-risk AI system on the market or putting it into service and that are appropriate to be implemented by the deployer." }, { "requirementID": "15.1 Accuracy, Robustness and Cybersecurity", "requirementText": "High-risk AI systems shall be designed and developed in such a way that they achieve an appropriate level of accuracy, robustness, and cybersecurity, and that they perform consistently in those respects throughout their lifecycle." }, { "requirementID": "17.1 Quality Management Systems", "requirementText": "Providers of high-risk AI systems shall put a quality management system in place that ensures compliance with this Regulation. That system shall be documented in a systematic and orderly manner in the form of written policies, procedures and instructions, and shall include at least the following aspects:\n(a) a strategy for regulatory compliance, including compliance with conformity assessment procedures and procedures for the management of modifications to the high-risk AI system;\n(b) techniques, procedures and systematic actions to be used for the design, design control and design verification of the high-risk AI system;\n(c) techniques, procedures and systematic actions to be used for the development, quality control and quality assurance of the high-risk AI system;\n(d) examination, test and validation procedures to be carried out before, during and after the development of the high-risk AI system, and the frequency with which they have to be carried out;\n(e) technical specifications, including standards, to be applied and, where the relevant harmonised standards are not applied in full or do not cover all of the relevant requirements set out in Section 2, the means to be used to ensure that the high-risk AI system complies with those requirements.\n(f) systems and procedures for data management, including data acquisition, data collection, data analysis, data labelling, data storage, data filtration, data mining, data aggregation, data retention and any other operation regarding the data that is performed before and for the purpose of the placing on the market or the putting into service of high-risk AI systems;\n(g) the risk management system referred to in Article 9;\n(h) the setting-up, implementation and maintenance of a post-market monitoring system, in accordance with Article 72;\n(i) procedures related to the reporting of a serious incident in accordance with Article 73;\n(j) the handling of communication with national competent authorities, other relevant authorities, including those providing or supporting the access to data, notified bodies, other operators, customers or other interested parties;\n(k) systems and procedures for record-keeping of all relevant documentation and information;\n(l) resource management, including security-of-supply related measures;\n(m) an accountability framework setting out the responsibilities of the management and other staff with regard to all the aspects listed in this paragraph." }, { "requirementID": "17.2 Quality Management Systems", "requirementText": "The implementation of the aspects referred to in paragraph 1 shall be proportionate to the size of the provider’s organisation. Providers shall, in any event, respect the degree of rigour and the level of protection required to ensure the compliance of their high-risk AI systems with this Regulation." }, { "requirementID": "17.4 Quality Management Systems", "requirementText": "For providers that are financial institutions subject to requirements regarding their internal governance, arrangements or processes under Union financial services law, the obligation to put in place a quality management system, with the exception of paragraph 1, points (g), (h) and (i) of this Article, shall be deemed to be fulfilled by complying with the rules on internal governance arrangements or processes pursuant to the relevant Union financial services law. To that end, any harmonised standards referred to in Article 40 shall be taken into account." }, { "requirementID": "50.1 Transparency obligations for providers and deployers of certain AI systems", "requirementText": "Providers shall ensure that AI systems intended to interact directly with natural persons are designed and developed in such a way that the natural persons concerned are informed that they are interacting with an AI system, unless this is obvious from the point of view of a natural person who is reasonably well-informed, observant and circumspect, taking into account the circumstances and the context of use. This obligation shall not apply to AI systems authorised by law to detect, prevent, investigate or prosecute criminal offences, subject to appropriate safeguards for the rights and freedoms of third parties, unless those systems are available for the public to report a criminal offence." } ] } }, "European Commission": { "Assessment List for Trustworthy Artificial Intelligence (ALTAI)": { "link": "https://digital-strategy.ec.europa.eu/en/library/assessment-list-trustworthy-artificial-intelligence-altai-self-assessment", "requirements": [ { "requirementID": "REQUIREMENT #2 Technical Robustness and Safety", "requirementText": "A crucial requirement for achieving Trustworthy AI systems is their dependability (the ability to deliver services that can justifiably be trusted) and resilience (robustness when facing changes). Technical robustness requires that AI systems are developed with a preventative approach to risks and that they behave reliably and as intended while minimising unintentional and unexpected harm as well as preventing it where possible. This should also apply in the event of potential changes in their operating environment or the presence of other agents (human or artificial) that may interact with the AI system in an adversarial manner. The questions in this section address four main issues: 1) security; 2) safety; 3) accuracy; and 4) reliability, fall-back plans and reproducibility." }, { "requirementID": "REQUIREMENT #3 Privacy and Data Governance", "requirementText": "Closely linked to the principle of prevention of harm is privacy, a fundamental right particularly affected by AI systems. Prevention of harm to privacy also necessitates adequate data governance that covers the quality and integrity of the data used, its relevance in light of the domain in which the AI systems will be deployed, its access protocols and the capability to process data in a manner that protects privacy." } ] }, "Ethics guidelines for trustworthy AI": { "link": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirements": [ { "requirementID": "1.2.4 Reliability and Reproducability", "requirementText": "It is critical that the results of AI systems are reproducible, as well as reliable. A reliable AI system is one that works properly with a range of inputs and in a range of situations. This is needed to scrutinise an AI system and to prevent unintended harms. Reproducibility describes whether an AI experiment exhibits the same behaviour when repeated under the same conditions. This enables scientists and policy makers to accurately describe what AI systems do. Replication files can facilitate the process of testing and reproducing behaviours." }, { "requirementID": "1.5.1 Avoidance of unfair bias", "requirementText": "Identifiable and discriminatory bias should be removed in the collection phase where possible. The way in which AI systems are developed (e.g. algorithms’ programming) may also suffer from unfair bias. This could be counteracted by putting in place oversight processes to analyse and address the system’s purpose, constraints, requirements and decisions in a clear and transparent manner. Moreover, hiring from diverse backgrounds, cultures and disciplines can ensure diversity of opinions and should be encouraged." }, { "requirementID": "2.2.3 Standardisation", "requirementText": "Standards, for example for design, manufacturing and business practices, can function as a quality management system for AI users, consumers, organisations, research institutions and governments by offering the ability to recognise and encourage ethical conduct through their purchasing decisions. Beyond conventional standards, coregulatory approaches exist: accreditation systems, professional codes of ethics or standards for fundamental rights compliant design. Current examples are e.g. ISO Standards or the IEEE P7000 standards series, but in the future a possible ‘Trustworthy AI' label might be suitable, confirming by reference to specific technical standards that the system, for instance, adheres to safety, technical robustness and transparency." } ] } }, "Federal Office for Information Security": { "AI Security Concerns in a Nutshell": { "link": "https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/KI/Practical_Al-Security_Guide_2023.pdf?__blob=publicationFile&v=5", "requirements": [ { "requirementID": "3.3 Defending against Evasion Attacks - Adversarial Retraining", "requirementText": "Adversarial retraining consists of iteratively generating adversarial examples and repeatedly training the model on them. As a result, the robustness of the model against the selected attack methods increases." }, { "requirementID": "5.3 Defending against Poisoning and Backdoor Attacks - Retraining", "requirementText": "For network pruning, benign data samples are fed into the trained neural network, and their average activation is measured. Neurons without a high level of activation can be trimmed without substantially reducing the model’s accuracy. In the process, potential backdoors can be removed as well. Similar to retraining, the complete success of the measure cannot be guaranteed [20]." }, { "requirementID": "5.3 Defending against Poisoning and Backdoor Attacks - Autoencoder Detection", "requirementText": "Regularization can lower the success rate of backdoor attacks without significantly degrading the baseline performance on benign inputs [18]." } ] } }, "Google": { "Secure AI Framework": { "link": "https://www.saif.google/secure-ai-framework", "requirements": [ { "requirementID": "Privacy Enhancing Technologies", "requirementText": "Use technologies that minimize, de-identify, or restrict use of PII data in training or evaluating models." }, { "requirementID": "Input Validation and Sanitization", "requirementText": "Block or restrict adversarial queries to AI models." }, { "requirementID": "Adversarial Training and Testing", "requirementText": "Use techniques to make AI models robust to adversarial inputs (i.e. prompts) in the context of their use in applications." }, { "requirementID": "Agent Permissions", "requirementText": "Use least-privilege principle as the upper bound on agentic system permissions to minimize the number of tools that an agent is permitted to interact with and the actions it is allowed to take. An agentic system’s use of privileges should be contextual and dynamic, adapting to the specific user query and trusted contextual information. This design also applies to agents that have access to user information. For example, an agent asked to fill out a form or answer questions should share only contextually appropriate information and can be designed to dynamically minimize exposed data using reference monitors." } ] } }, "IBM": { "IBM Framework for Securing Generative AI": { "link": "https://www.ibm.com/products/tutorials/ibm-framework-for-securing-generative-ai", "requirements": [ { "requirementID": "Secure the usage", "requirementText": "During inferencing and live use, attackers can manipulate prompts to jailbreak guardrails and coax models into misbehaving by generating disallowed responses to harmful prompts that include biased, false and other toxic information. This can inflict reputational damage on the enterprise. Attackers might also seek to manipulate the model and analyze input/output pairs to train a surrogate model to mimic the behavior of the target model, effectively “stealing” its capabilities and costing an enterprise its competitive advantage.\n\nSeveral types of attacks are concerning in this stage of the AI pipeline. First, prompt injections—where attackers use malicious prompts to jailbreak models and get unwarranted access, steal sensitive data or introduce bias into outputs. Another concern involves model denial of service, where attackers overwhelm the LLM with inputs that degrade the quality of service and incur high resource costs. Organizations should also prepare for and defend against model theft, where attackers craft inputs to collect model outputs to train a surrogate model that mimics the behavior of the target model.\n\nOur best practices include monitoring for malicious inputs such as prompt injections and outputs containing sensitive data or inappropriate content, and implementing new defenses that can detect and respond to AI-specific attacks such as data poisoning, model evasion and model extraction. New AI-specific solutions have entered the market under the name of machine learning detection and response (MLDR). Alerts generated from these solutions can be integrated into security operations solutions, such as IBM Security® QRadar®, enabling security operations center (SOC) teams to quickly initiate response playbooks that deny access, quarantine or disconnect compromised models." } ] } }, "ICO": { "Guidance on the AI Auditing Framework - Draft guidance for consultation ": { "link": "https://ico.org.uk/media2/about-the-ico/consultations/2617219/guidance-on-the-ai-auditing-framework-draft-for-consultation.pdf", "requirements": [ { "requirementID": "What should we assess in our DPIA?", "requirementText": "Your DPIA needs to describe the nature, scope, context and purposes of any processing of personal data - it needs to make clear how and why you are going to use AI to process the data. You need to detail:\n• how you will collect, store and use data;\n• the volume, variety and sensitivity of the data;\n• the nature of your relationship with individuals; and\n• the intended outcomes for individuals or wider society, as well as for you.\nIn the context of the AI lifecycle, a DPIA will best serve its purpose if you undertake it at the earliest stages of project development. It should feature, at a minimum, the following key components. " }, { "requirementID": "Why is controllership important for AI systems?", "requirementText": "In many cases, the various processing operations involved in AI may be undertaken by a number of different organisations. It is therefore crucial that you determine who is a controller, joint controller or processor if your use of AI involves multiple organisations.\nA first step to understanding these relationships is to identify the distinct sets of processing operations and their purposes (see ‘What should we assess in our DPIA?). For each of these, you and the organisations you work with then need to assess whether you are a controller, processor or joint controller. You should consult our guidance on controller/processor for assistance in this assessment, but in essence:\n• if you decide on the purposes and means of processing, you are a controller;\n• if you process personal data under the instruction of another organisation, you are a processor; and\n• if you jointly determine the purposes and means of processing with another organisation, you are joint controllers. " }, { "requirementID": "What type of decisions may make us a controller?", "requirementText": "The type of decisions that are likely to make you a controller include deciding:\n• to collect personal data in the first place\n• the purpose(s) for that processing\n• which individuals to collect the data about, and what to tell them about the processing;\n• how long you will retain the data for; and\n• how to respond to requests made in line with individuals’ rights.\n\nFor more specifics on the circumstances that may make you a controller, read our guidance on controllers and processors. In the context of AI, the type of decisions made by controllers can include:\n• the source and nature of the data used to train an AI model;\n• the target output of the model (ie what is being predicted or classified);\n• the broad kinds of ML algorithms that will be used to create models from the data (eg regression models, decision trees, random forests, neural networks);\n• feature selection – the features that may be used in each model;\n• key model parameters (eg how complex a decision tree can be, or how many models will be included in an ensemble);\n• key evaluation metrics and loss functions, such as the trade-off between false positives and false negatives; and\n• how any models will be continuously tested and updated: how often, using what kinds of data, and how ongoing performance will be assessed. " }, { "requirementID": "What are our security requirements?", "requirementText": "There is no “one-size-fits-all” approach to security. The appropriate security measures you should adopt depend on the level and type of risks that arise from specific processing activities.\nUsing AI to process any personal data has important implications for your security risk profile, and you need to assess and manage these carefully.\nSome implications may be triggered by the introduction of new types of risks, eg adversarial attacks on machine learning models. " }, { "requirementID": "Preventative Controls - 2", "requirementText": "Comply to or assess an AI system against external security certifications or schemes." }, { "requirementID": "Preventative Controls - 3", "requirementText": "Subject software to a quality review where one or more individuals view and read parts of its source code. At least one of the reviewers must not be the author of the code." }, { "requirementID": "Preventative Controls - 11", "requirementText": "Assess more secure implementations of the trained model, and implement them as appropriate, post development but pre-deployment." } ] } }, "IETF": { "Security Requirements for AI Agents": { "link": "https://www.ietf.org/archive/id/draft-ni-a2a-ai-agent-security-requirements-00.html", "requirements": [ { "requirementID": "5.3. Converting to Internal Workflow", "requirementText": "Workflow Generation: Complex tasks often require multi-agent collaboration. The master agent receives, parses, and extracts the original job request from the external requesting agent, then create sequential workflows or parallel calls. This requires the master agent to have information of all callable internal API assets, agent capabilities, etc.\n\nDownscoping: If the master agent intends to use a workflow, it extracts the original caller's identity and authorization context, and initiates a new internal workflow. It should follow the current least privilege best practice of downscoping-Transaction Tokens as specified in [I-D.draft-tulshibagwale-oauth-transaction-tokens-05]. The access rights to each downstream workload decrease." } ] } }, "IMDA": { "Model AI Governance Framework for Agentic AI": { "link": "https://www.imda.gov.sg/-/media/imda/files/about/emerging-tech-and-research/artificial-intelligence/mgf-for-agentic-ai.pdf", "requirements": [ { "requirementID": "2.1.2 Bound risks through design by defining agents limits and permissions - Agent limits", "requirementText": "Having selected an appropriate agent use case, organisations can further bound the risks by defining appropriate limits and permission policies for each agent.\nOrganisations should consider defining limits on:\n• Agent’s access to tools and systems: Define policies that give agents only the minimum tools and data access needed for it to complete its task. For example, a coding assistant may not require access to a web search tool, especially if it already has curated access to the latest software documentation.\n• Agent’s autonomy: For process-driven tasks, SOPs and protocols are frequently used to improve consistency and reduce unpredictability. Define similar SOPs for agentic workflows that an agent is constrained to follow, rather than giving the agent the freedom to define every step of the workflow.\n• Agent’s area of impact: Design mechanisms and procedures to take agents offline and limit their potential scope of impact when they malfunction. This can include running agents in self-contained environments with limited network and data access, particularly when they are carrying out high-risk tasks such as code execution." }, { "requirementID": "2.3.1 During design and development, use technical controls", "requirementText": "Organisations should design and implement technical controls in the agentic AI system to mitigate identified risks. For agents specifically, in addition to baseline software and LLM controls, consider adding controls for:\n• New agentic components, such as planning and reasoning and tools\n• Increased security concerns from the larger attack surface and new protocols\nFor illustration, these are some sample controls for agents. For a more comprehensive list, organisations can refer to CSA’s Draft Addendum on Securing Agentic AI and GovTech’s Agentic Risk and Capability Framework." } ] } }, "ISO": { "42001:2023 - Information technology — Artificial intelligence — Management system": { "link": "https://www.iso.org/standard/42001", "requirements": [ { "requirementID": "4.1", "requirementText": "Understanding the organization and its context" }, { "requirementID": "4.2", "requirementText": "Understanding the needs and expectations of interested parties" }, { "requirementID": "4.3", "requirementText": "Determining the scope of the AI management system" }, { "requirementID": "6.2", "requirementText": "AI objectives and planning to achieve them" } ] } }, "ISO/IEC": { "DIS 27090": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:27090:dis:ed-1:v1:en", "requirements": [ { "requirementID": "7.2", "requirementText": "Conflicting interactions of mitigations" } ] }, "TR 27091": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:27091:dis:ed-1:v1:en", "requirements": [ { "requirementID": "6.2", "requirementText": "Privacy threats to AI models" }, { "requirementID": "7.2.1", "requirementText": "Privacy engineering" } ] }, "TR 27563:2023": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:tr:27563:ed-1:v1:en", "requirements": [ { "requirementID": "7.2", "requirementText": "Provide assessment of systems of interest" } ] } }, "METI (Japan)": { "Governance Guidelines for Implementation of AI Principles": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20220128_2.pdf", "requirements": [ { "requirementID": "Action Target 3-4", "requirementText": "Companies that develop and operate AI systems and those that provide data should, under the leadership of top management, reduce incident-related burdens on users by preventing incidents and through early response." }, { "requirementID": "Action Target 5-1", "requirementText": "Companies that develop and operate AI systems should, under the leadership of top management, have individuals independent of the design and operation of the AI management system verify whether an AI management system such as a gap analysis process is appropriately designed and operated in light of the AI governance goals, in other words, whether an AI management system appropriately works for the achievement of the AI governance goals through the implementation of Action Targets 3 and 4." } ] } }, "MIC/METI (Japan)": { "AI Guidelines for Business": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirements": [ { "requirementID": "Human-Centric - 3", "requirementText": "Generative AI has enabled everyone to forge fake information that seems to be true and fair, so recognize the increasing risk of destabilizing and confusing the society through disinformation, misinformation, and biased information generated by AI, and take necessary countermeasures." }, { "requirementID": "Safety - 2", "requirementText": "Develop, provide, or use AI systems and services within the range in which the AI business actor can control, preventing damage due to a provision or use that deviates from the intended purpose." }, { "requirementID": "Fairness - 1 (a)", "requirementText": "There are a broad range of factors that can produce an inappropriate bias, so identify the factors that might produce biases that can be considered as problems from the viewpoint of fairness. Those factors may include technological elements (training data, AI model training process, prompts entered by AI business users or non-business users , and reference information and collaborating external services used by AI models for inference) and behaviors of AI business users." }, { "requirementID": "Ensuring security - 2 (a)", "requirementText": "To maintain the confidentiality, integrity, and availability of AI systems and services and ensure safe and secure AI use constantly, take reasonable measures based on the technological level at the time." }, { "requirementID": "Ensuring security - 1 (b)", "requirementText": "Understand the characteristics of AI systems and services, and examine whether the inter-system connections necessary for normal operations are properly established." } ] } }, "Microsoft": { "Cloud Adoption Framework - Secure AI": { "link": "https://learn.microsoft.com/en-us/azure/cloud-adoption-framework/scenarios/ai/secure", "requirements": [ { "requirementID": "Secure AI resources\n3 - Apply platform-specific security controls", "requirementText": "Different AI deployment models face distinct security threats based on their architecture and exposure points. Platform-tailored controls address the specific vulnerabilities present in each deployment type. Follow dedicated security guidance based on your deployment model:\nAzure PaaS security\nAzure IaaS security" } ] }, "Responsible AI Standard": { "link": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirements": [ { "requirementID": "A3.1", "requirementText": "Document in the Impact Assessment how the system’s use will solve the problem posed by each intended use, recognizing that there may be multiple valid ways in which to solve the problem." }, { "requirementID": "A3.2", "requirementText": "Define and document for each model in the AI system:\n1) the model’s proposed inputs and how well they represent the concepts they are intended to represent; include analysis of the limitations of this representation,\n2) the model’s proposed output and how well it represents the concept it is intended to represent; include analysis of the limitations of this representation, and\n3) limitations to the generalizability of the resulting model based on the training and testing data that will be used." }, { "requirementID": "A3.3", "requirementText": "Define and document Responsible Release Criteria for this Goal. Include:\n1) a concise definition of the problem being solved in the intended use,\n2) performance metrics and their Responsible Release Criteria, and\n3) error types and their Responsible Release Criteria." }, { "requirementID": "A3.6", "requirementText": "Provide documentation to customers which describes the system’s:\n1) intended uses, and\n2) evidence that the system is fit for purpose for each intended use.\nWhen the system is a platform service made available to external customers or partners, include this information in the required Transparency Note." }, { "requirementID": "A4.1", "requirementText": "Define and document data requirements with respect to the system’s intended uses, stakeholders, and the geographic areas where the system will be deployed. Document these requirements in the Impact Assessment." }, { "requirementID": "T1.6", "requirementText": "If there are Responsible Release Criteria for metrics or rubrics that that have not been met, consult with the reviewers named in the Impact Assessment, and in the case of Sensitive Uses, with the Office of Responsible AI, to develop a plan detailing how the gap will be managed until it can be closed. Document that plan." }, { "requirementID": "F1.6", "requirementText": "Reassess the system design, including the choice of training data, features, objective function, and training algorithm, to pursue the goals of:\n1) improving performance for any identified demographic group that does not meet any target minimum performance level, and\n2) minimizing performance differences between identified demographic groups, paying particular attention to those that exceed the target maximum, while recognizing that doing so may appear to affect system performance and that it is seldom clear how to make such tradeoffs.\nConsult with your attorney to determine your approach to this, including how you will identify and document tradeoffs." }, { "requirementID": "F1.7", "requirementText": "Identify and document any justifiable factors, such as circumstantial and other operational factors (e.g., “background noise” for speech recognition systems or “image resolution” for facial recognition systems), that account for:\n1) any inability to meet any target minimum performance level for any identified demographic group, and\n2) any remaining performance differences between identified demographic groups." }, { "requirementID": "F2.6", "requirementText": "Reassess the system design, including the choice of training data, features, objective function, and training algorithm, to pursue the goal of minimizing differences between the rates at which resources and opportunities are allocated to identified demographic groups, paying particular attention to those that exceed the target maximum difference, while recognizing that doing so may appear to affect system performance and it is seldom clear how to make such trade-offs.\nConsult with your attorney to determine your approach to this, including how you will identify and document trade-offs." }, { "requirementID": "F3.5", "requirementText": "Reassess the system design, including the choice of training data, features, objective function, and training algorithm, to pursue the goal of minimizing the potential for stereotyping, demeaning, and erasing the identified demographic groups." }, { "requirementID": "RS1.5", "requirementText": "Define intended uses, if any, where additional operational factors, more narrow or different acceptable ranges, or lower acceptable error rates (including false positive and false negative error rates), are advised to ensure reliability and safety. Document your conclusions." } ] } }, "MITRE": { "ATLAS Framework": { "link": "https://atlas.mitre.org/mitigations", "requirements": [ { "requirementID": "AML.M0006 - Use Ensemble Methods", "requirementText": "Use an ensemble of models for inference to increase robustness to adversarial inputs. Some attacks may effectively evade one model or model family but be ineffective against others." }, { "requirementID": "AML.M0009 - Use Multi-Modal Sensors", "requirementText": "Incorporate multiple sensors to integrate varying perspectives and modalities to avoid a single point of failure susceptible to physical attacks." }, { "requirementID": "AML.M0020 - Generative AI Guardrails", "requirementText": "Guardrails are safety controls that are placed between a generative AI model and the output shared with the user to prevent undesired inputs and outputs. Guardrails can take the form of validators such as filters, rule-based logic, or regular expressions, as well as AI-based approaches, such as classifiers and utilizing LLMs, or named entity recognition (NER) to evaluate the safety of the prompt or response. Domain specific methods can be employed to reduce risks in a variety of areas such as etiquette, brand damage, jailbreaking, false information, code exploits, SQL injections, and data leakage." }, { "requirementID": "AML.M0021 - Generative AI Guidelines", "requirementText": "Guidelines are safety controls that are placed between user-provided input and a generative AI model to help direct the model to produce desired outputs and prevent undesired outputs.\n\nGuidelines can be implemented as instructions appended to all user prompts or as part of the instructions in the system prompt. They can define the goal(s), role, and voice of the system, as well as outline safety and security parameters." }, { "requirementID": "AML.M0022 - Generative AI Model Alignment", "requirementText": "When training or fine-tuning a generative AI model it is important to utilize techniques that improve model alignment with safety, security, and content policies.\n\nThe fine-tuning process can potentially remove built-in safety mechanisms in a generative AI model, but utilizing techniques such as Supervised Fine-Tuning, Reinforcement Learning from Human Feedback or AI Feedback, and Targeted Safety Context Distillation can improve the safety and alignment of the model." }, { "requirementID": "AML.M0026 - Privileged AI Agent Permissions Configuration", "requirementText": "AI agents may be granted elevated privileges above that of a normal user to enable desired workflows. When deploying a privileged AI agent, or an agent that interacts with multiple users, it is important to implement robust policies and controls on permissions of the privileged agent. These controls include Role-Based Access Controls (RBAC), Attribute-Based Access Controls (ABAC), and the principle of least privilege so that the agent is only granted the necessary permissions to access tools and resources required to accomplish its designated task(s)." }, { "requirementID": "AML.M0027 - Single-User AI Agent Permissions Configuration", "requirementText": "When deploying an AI agent that acts as a representative of a user and performs actions on their behalf, it is important to implement robust policies and controls on permissions and lifecycle management of the agent. Lifecycle management involves establishing identity, protocols for access management, and decommissioning of the agent when its role is no longer needed. Controls should also include the principle of least privilege and delegated access from the user account. When acting as a representative of a user, the AI agent should not be granted permissions that the user would not be granted within the system or organization." }, { "requirementID": "AML.M0028 - AI Agent Tools Permissions Configuration", "requirementText": "When deploying tools that will be shared across multiple AI agents, it is important to implement robust policies and controls on permissions for the tools. These controls include applying the principle of least privilege along with delegated access, where the tools receive the permissions, identities, and restrictions of the AI agent calling them. These configurations may be implemented either in MCP servers which connect the agents to the tools calling them or, in more complex cases, directly in the configuration files of the tool." }, { "requirementID": "AML.M0030 - Restrict AI Agent Tool Invocation on Untrusted Data", "requirementText": "Untrusted data can contain prompt injections that invoke an AI agent's tools, potentially causing confidentiality, integrity or availability violations. It is recommended that tool invocation be restricted or limited when untrusted data enters the LLM's context.\n\nThe degree to which tool invocation is restricted may depend on the potential consequences of the action. Consider blocking the automatic invocation of tools or requiring user confirmation once untrusted data enters the LLM's context. For high consequence actions, consider always requiring user confirmation." }, { "requirementID": "AML.M0031 - Memory Hardening", "requirementText": "Memory Hardening involves developing trust boundaries and secure processes for how an AI agent stores and accesses memory and context. This may be implemented using a combination of strategies including restricting an agent's ability to store memories by requiring external authentication and validation for memory updates, performing semantic integrity checks on retrieved memories before agents execute actions, and implementing controls for monitoring of memory and remediation processes for poisoned memory." } ] }, "SAFE-AI": { "link": "https://atlas.mitre.org/pdf-files/SAFEAI_Full_Report.pdf", "requirements": [ { "requirementID": "Unauthorized access to environment, platform/tool", "requirementText": "Malicious actors can exploit unauthorized access to perturb valid inputs to AI models, causing them to consistently generate incorrect decisions. If safeguards are not in place to validate inputs, AI-enabled systems may be vulnerable to attacks that inject instructions or commands to an AI model, causing it to execute unauthorized tasks or generate erroneous outputs. Also, be wary of “off-label use” where an AI component is developed outside of an organization’s security safeguards, or a component has been lifted from one context or application and then “fine-tuned” to be used in a different setting." }, { "requirementID": "Insecure deserialization – embedding and executing remote unapproved code or other malicious\nactivities", "requirementText": "Malicious users can sometimes exert control over an AI enabled system by finding a command sequence that abuses the logic of the system and causes it to execute unauthorized tasks or generate incorrect behavior. The most prominent recent examples of this are the prompt attacks that some large language models are vulnerable to. Additionally, some AI models use procedural representations of knowledge and models (e.g., as in a rule-based system). These representations might be compromised by an adversary to enable the execution of malicious code. All procedural data and input data must be rigorously sanitized and validated." }, { "requirementID": "Backdoor and malware insertion", "requirementText": "Given the tendency of AI-enabled systems to depend on massive data stores and somewhat complex models and decision logic, it can be difficult identify all unsecured points of entry vulnerable to backdoor and malware insertion attacks. Attackers can manipulate data in all phases of the system lifecycle, exploit vulnerabilities in AI algorithms and models, or use a variety of other techniques to insert backdoors into AI systems that get triggered once the AI is deployed. It is important to anticipate potential threats and AI-related attack surfaces during the design phase, secure and verify data and software during development, and establish testing procedures to regularly monitor system components for data/model drift, changes in performance, or other AI system behavior issues once it is deployed. If a potential attack is detected, information about errors and attack patterns must be shared with incident databases." }, { "requirementID": "Indirect Prompt Injection", "requirementText": "Adversaries may devise malicious prompts that cause the AI component to act in unintended ways. The attacks may be designed to bypass defenses or allow the adversary to issue privileged commands. The attack is indirect when the AI component ingests the malicious prompt from a separate data source (e.g., text or multimedia from a website, chat plugins) as part of its normal operation. Plugins may be vulnerable to an indirect prompt injection attack that uses the AI component to exfiltrate the history of a user conversation with an external website. The user may never be aware of the prompt injection. This type of injection can be used by the adversary to target the PII of the user." }, { "requirementID": "Direct Prompt Injection", "requirementText": "Adversaries may devise malicious prompts to an AI component causing it to act in unintended ways. Direct prompt injections are often an attempt to manipulate the AI component to generate harmful content or issue privileged commands to gain a foothold on the system, including placing AI component in a state in which it will freely respond to any user input, bypassing controls or guardrails placed on the AI component." }, { "requirementID": "Excessive Agency", "requirementText": "Excessive Agency refers to situations where AI components have access to APIs, plugins, extensions, and tools with capabilities that go beyond what is necessary to support the AI component operations. Excessive permissions, unnecessary functionality, and unchecked authority to act autonomously are all examples of excessive agency that can result in unintended and unacceptable application behaviors with potentially damaging consequences. To mitigate these risks, developers need to limit extension capabilities (functionality, permissions, and autonomy) to only what is absolutely necessary, track user authorization, require human approval for all actions, and implement authorization in downstream systems." }, { "requirementID": "Content Manipulation", "requirementText": "Content manipulation poses a significant threat to Google DocumentAI, particularly in the context of OCR. Malicious actors can intentionally alter documents by changing numbers or formatting to deceive the AI, resulting in errors and misclassifications. Subtle content alterations can result in incorrect data extraction and faulty decision-making, compromising the integrity of the documents processed by the AI. By deceiving AI-enable systems via document content manipulation, attackers can undermine trust in the automated document processing system." }, { "requirementID": "Evade AI model", "requirementText": "Adversaries can craft input data designed to prevent AI models from correctly identifying the contents of the data. For example, an adversary might introduce subtle perturbations that cause the model to misclassify or overlook meaningful information. This technique can be used to evade downstream tasks where machine learning is utilized by exploiting weaknesses in the AI model algorithms. Additionally, the adversary may evade machine learningbased virus/malware detection or network scanning tools towards the goal of a traditional cyber-attack." }, { "requirementID": "Robotic Process Automation Permissions", "requirementText": "Robotic Process Automation (RPA) bots are responsible for handling and manipulating sensitive data. If access controls and policies are not properly implemented, the bots can cause damage to systems and data due to errors. RPA bots are vulnerable to adversarial attacks, such as Evasion Attack, where malicious actors manipulate input data to deceive the bots, causing them to perform unintended actions, misclassify data, or corrupt data. Monitoring unauthorized access and insider threats is crucial, as bots with excessive permissions can be misused for malicious purposes. Ensuring robust security measures and continuous monitoring can mitigate these risks and protect sensitive data." } ] } }, "Multi Agency": { "Guidelines for secure AI system development": { "link": "https://www.ncsc.gov.uk/files/Guidelines-for-secure-AI-system-development.pdf", "requirements": [ { "requirementID": "Design your system for security as well as functionality and performance", "requirementText": "You are confident that the task at hand is most appropriately addressed using AI. Having determined this, you assess the appropriateness of your AI-specific design choices. You consider your threat model and associated security mitigations alongside functionality, user experience, deployment environment, performance, assurance, oversight, ethical and legal requirements, among other considerations." }, { "requirementID": "Consider security benefits and trade-offs when selecting your AI model", "requirementText": "Your choice of AI model will involve balancing a range of requirements. This includes choice of model architecture, configuration, training data, training algorithm and hyperparameters. Your decisions are informed by your threat model, and are regularly reassessed as AI security research advances and understanding of the threat evolves." }, { "requirementID": "Collect and share lessons learned", "requirementText": "You participate in information-sharing communities, collaborating across the global ecosystem of industry, academia and governments to share best practice as appropriate. You maintain open lines of communication for feedback regarding system security, both internally and externally to your organisation, including providing consent to security researchers to research and report vulnerabilities. When needed, you escalate issues to the wider community, for example publishing bulletins responding to vulnerability disclosures, including detailed and complete common vulnerability enumeration. You take action to mitigate and remediate issues quickly and appropriately." } ] } }, "NCSC/NSA/CISA etc": { "AI Data Security\n": { "link": "https://media.defense.gov/2025/May/22/2003720601/-1/-1/0/CSI_AI_DATA_SECURITY.PDF", "requirements": [ { "requirementID": "3.4 Ensemble Methods/ collaborative learning ", "requirementText": "Implement collaborative learning frameworks that combine an ensemble of multiple, distinct AI models to reach a consensus on output predictions. This approach can help counteract the impact of data poisoning, since malicious inputs may only affect a subset of the collaborative models, allowing the majority to maintain accuracy and reliability." } ] } }, "NIST": { "AI 100-2e2025: Adversarial Machine Learning\nA Taxonomy and Terminology of Attacks and Mitigations": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-2e2025.pdf", "requirements": [ { "requirementID": "2.2.5 Adversarial training", "requirementText": "Introduced by Goodfellow et al. [144] and further developed by Madry et al. [232], adversarial training is a general method that augments training data with adversarial examples generated iteratively during training using their correct labels. The stronger the adversarial attacks for generating adversarial examples are, the more resilient the trained model becomes. Adversarial training results in models with more semantic meaning than standard models[379], but this benefit usually comes at the cost of decreased model accuracy on clean data. Additionally, adversarial training is expensive due to the iterative generation of adversarial examples during training." }, { "requirementID": "2.2.5 Randomized smoothing", "requirementText": "Proposed by Lecuyer et al. [207] and further improved by 17 NIST AI 100-2e2025 March 2025 Cohen et al. [94], randomized smoothing is a method that transforms any classifier into a certifiable robust smooth classifier by producing the most likely predictions under Gaussian noise perturbations. This method results in provable robustness for `2 evasion attacks, even for classifiers trained on large-scale datasets, such as ImageNet. Randomized smoothing typically provides certified prediction to a subset of testing samples, the exact number of which depends on factors such as the size of the potential perturbations or the characteristics of the training data and model. Recent results have extended the notion of certified adversarial robustnessto `2-norm bounded perturbations by combining a pretrained denoising diffusion probabilistic model and a standard high-accuracy classifier [62]. Li et al. [211] developed a taxonomy for the robustness verification and training of representative algorithms. They also revealed the characteristics, strengths, limitations, and fundamental connections among these approaches, along with theoretical barriers facing the field." }, { "requirementID": "2.2.5 Formal verification", "requirementText": "Another method for certifying the adversarial robustness of a neural network is based on techniques from FORMAL METHODS. Reluplex uses satisfiability modulo theories (SMT) solvers to verify the robustness of small feedforward neural networks[191]. AI2 isthe first verification method applicable to convolutional neural networks using abstract interpretation techniques [136]. These methods have been extended and scaled up to larger networks in follow-up verification systems, such as DeepPoly [346], ReluVal [394], and Fast Geometric Projections (FGP) [131]. Formal verification techniques have significant potential for certifying neural network robustness but are limited by their lack of scalability, computational cost, and restriction in the type of supported algebraic operations such as addition, multiplication, etc." }, { "requirementID": "2.3.1 Robust training", "requirementText": "An alternative approach to mitigating availability poisoning attacks is to modify the ML training algorithm to increase the robustness of the resulting model. The defender can train an ensemble of multiple models and generate predictions via model voting [37, 209, 395]. Several papers apply techniques from robust optimization,such as using a trimmed lossfunction [109, 179]. Rosenfeld et al. [314] proposed the use of randomized smoothing to add noise during training to provide protection against label-flipping attacks." }, { "requirementID": "2.3.3 Trigger reconstruction", "requirementText": "This class of mitigations aims to reconstruct the backdoor trigger, assuming that it is at a fixed location in the poisoned training samples. NeuralCleanse by Wang et al. [390] developed the first trigger reconstruction approach and used optimization to determine the most likely backdoor pattern that reliably misclassifies the test samples. The initial technique has been improved to reduce performance time on several classes and simultaneously support multiple triggers inserted into the model [163, 411]. A representative system in this class is Artificial Brain Simulation (ABS) by Liu et al. [221], which stimulates multiple neurons and measures the activations to reconstruct the trigger patterns. Khaddaj et al. [193] developed a new primitive for detecting backdoor attacks and a corresponding effective detection algorithm with theoretical guarantees." }, { "requirementID": "2.3.3 Certified defenses", "requirementText": "Several methods for achieving certified defenses against data poisoning attacks have been proposed in the literature. BagFlip [440] is a modelagnostic defense that extends randomized smoothing [94] and combines training data bagging with adding noise to both training and testing samples. Deep Partition Aggregation [209] and Deep Finite Aggregation [396] are certified defensesthat partition the training data into disjointed subsets and train an ensemble method on each partition to reduce the impact of poisoned samples. Recently, FCert [398] provides a certified defense against data poisoning in few-shot classification settings used for both vision and text data." }, { "requirementID": "2.4.5 Data Reconstruction", "requirementText": "The discovery of reconstruction attacks against aggregate information motivated the rigorous definition of differential privacy (DP) [112, 113], an extremely strong definition of privacy that guarantees a bound on how much an attacker with access to the algorithm output can learn about each individual record in the dataset. The original pure definition of DP has a privacy parameter ε (i.e., privacy budget), which boundsthe probability thatthe attacker with access to the algorithm’s output can determine whether a particular record was included in the dataset. DP has been extended to the notions of approximate DP, which includes a second parameter δ that is interpreted as the probability of information accidentally being leaked in addition to ε and Rènyi DP [246]." }, { "requirementID": "3.2.3 Data Poisoning Attacks", "requirementText": "GenAI poisoning mitigations largely overlap with PredAI poisoning mitigations (see Sec.2.3). For preventing data poisoning with web-scale data dependencies, this includes verifying web downloads as a basic integrity check to ensure that domain hijacking has not injected new sources of data into the training dataset [57]. That is, the provider publishes cryptographic hashes, and the downloader verifies the training data. Data filtering can also attempt to remove poisoned samples, though detecting poisoned data within a large training corpus may be very difficult. While traditional software supply chain risk management practices such as vulnerability scanning of model artifacts can help manage some kinds of AI supply chain risks, new approaches are required to detect vulnerabilities in models such as those introduced through model poisoning attacks. Current proposed approaches include using methods from the field of mechanistic interpretability to identify backdoor features [67] and detecting and counteracting triggers when they are seen at inference time. Beyond these mitigations, risks can be reduced by understanding models as untrusted system components and designing applications such that risks from attacker-controlled model outputs are reduced [266]." }, { "requirementID": "3.3.3 Direct Prompting Attacks - Interventions during pre-training (2) and post-training (3)", "requirementText": "A range of training strategies have been proposed to increase the difficulty of accessing harmful model capabilities through direct prompt injection, including safety training during pre-training [197] or post-training [147, 445], adversarial training methods[340], and other methods to make jailbreak attacks more difficult [447]." }, { "requirementID": "3.3.3 Direct Prompting Attacks - Interventions during deployment (5) - Prompt instruction and formatting techniques", "requirementText": "Model instructions can cue the model to treat user input carefully, such as by wrapping user input in XML tags, appending specific instructions to the prompt, or otherwise attempting to clearly separate system instructions from user prompts [14, 206, 219]." }, { "requirementID": "3.4.4 Indirect Prompt Injection Attacks", "requirementText": "Various techniques (see Sec. 3.3.3) can be used throughout the development and deployment life cycle (Fig. 7) to mitigate attacks, including:\n• Several training techniques have been developed to mitigate against indirect prompt injection, including fine-tuning task-specific models[296] and training models to follow hierarchical trust relationships in prompts [387].\n• Detection schemes have been proposed to detect indirect prompt injection, and many LLM-based defenses have been designed to mitigate both direct and indirect prompt injection [6, 18, 154, 204, 313].\n• A range of input processing methods have been proposed to combat indirect prompt injection, including filtering out instructions from third-party data sources[146], designing promptsto help aid LLMs in separating trusted and untrusted data (i.e.,spotlighting [160, 206]), or instructing models to disregard instructions in untrusted data [206].\nMany of the defenses described in the context of direct prompt injection can also be adapted to mitigate indirect prompt injection. Because current mitigations do not offer full protection against all attacker techniques, application designers may design systems with the assumption that prompt injection attacks are possible if a model is exposed to untrusted input sources, such as by using multiple LLMs with different permissions[145, 405] or by allowing models to interact with potentially untrustworthy data sources only through well-defined interfaces[410]. Additionally, public education efforts can informmodel users and application designers of the risks of indirect prompt injection [266]." } ] }, "AI 800-1": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirements": [ { "requirementID": "Practice 1.1: Anticipate model capabilities - 2", "requirementText": "Consider also collecting evidence from less similar models, such as those that are significantly more or less powerful than the model (e.g., if a less capable model 15 presents a misuse risk, prioritize measuring for that risk)." }, { "requirementID": "Practice 1.1: Anticipate model capabilities - 3", "requirementText": "Consider other factors beyond model characteristics, such as resources available to develop the model, when relating the proxy models’ capabilities to anticipated ones." }, { "requirementID": "Practice 1.1: Anticipate model capabilities - 4", "requirementText": "Collect information about the prevalence of misuse, the usefulness of proxy models for potentially harmful or dual-use real-world tasks, and the efficacy of safeguards." }, { "requirementID": "Practice 2.2 Establish an organizational plan to manage misuse risk - 1", "requirementText": "Plan for information security and physical security practices necessary to manage misuse risk, when appropriate." }, { "requirementID": "Practice 2.2 Establish an organizational plan to manage misuse risk - 2", "requirementText": "Plan research needed to implement and evaluate adequate safeguards. If necessary, base these plans on the mapping established in Practice 2.1." }, { "requirementID": "Practice 2.2 Establish an organizational plan to manage misuse risk - 3", "requirementText": "Plan for decision-making about model deployment. This plan should include the organization’s plan for pre-deployment testing and determining which appropriate risk mitigations are available." }, { "requirementID": "Practice 3.2: Maintain security practices sufficient to prevent unauthorized access - 5", "requirementText": "Only develop a model that relies on confidentiality to manage misuse risk when the risk of threat actors gaining unauthorized access to the model is sufficiently mitigated. When the organization is made aware of an increased risk of unauthorized access, adjust or halt further development until the risk of unauthorized access is adequately managed." }, { "requirementID": "Practice 5.1: Implement safeguards proportionate to the model’s misuse risk - 2", "requirementText": "Consider implementing safeguards at various stages of model development, such as before, during, and after training and once the model has been integrated into a downstream system, as well as other mitigation measures. Some examples are included in Appendix B." }, { "requirementID": "Practice 5.2: Assess misuse risk based on implemented safeguards - 1", "requirementText": "Identify planned deployments that could impact misuse risk. For example, consider whether the planned deployment could impact the number of actors who have access to a model or the level of access that they would have." }, { "requirementID": "Practice 5.3: Adopt appropriate deployment strategies based on misuse risk assessments - 2", "requirementText": "Consider deployment strategies that provide additional real-world evidence to inform risk assessments without presenting significant misuse risks." }, { "requirementID": "Practice 6.2: Respond to incidents of model misuse - 3", "requirementText": "Plan for how identified instances of misuse will inform future development and deployment decisions, especially when reducing access to the model may not be possible." } ] }, "AI RMF 1.0": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirements": [ { "requirementID": "MAP 1.1", "requirementText": "Intended purposes, potentially beneficial uses, context- specific laws, norms and expectations, and prospective settings in which the AI system will be deployed are understood and documented. Considerations include: the specific set or types of users along with their expectations; potential positive and negative impacts of system uses to individuals, communities, organizations, society, and the planet; assumptions and related limitations about AI system purposes, uses, and risks across the development or product AI lifecycle; and related TEVV and system metrics." }, { "requirementID": "MAP 1.3", "requirementText": "The organization’s mission and relevant goals for AI technology are understood and documented." }, { "requirementID": "MAP 1.4", "requirementText": "The business value or context of business use has been clearly defined or – in the case of assessing existing AI systems – re-evaluated." }, { "requirementID": "MAP 1.6", "requirementText": "System requirements (e.g., “the system shall respect the privacy of its users”) are elicited from and understood by relevant AI actors. Design decisions take socio-technical implications into account to address AI risks." }, { "requirementID": "MAP 3.1", "requirementText": "Potential benefits of intended AI system functionality and performance are examined and documented." }, { "requirementID": "MAP 3.3", "requirementText": "Targeted application scope is specified and documented based on the system’s capability, established context, and AI system categorization." }, { "requirementID": "MANAGE 1.1", "requirementText": "A determination is made as to whether the AI system achieves its intended purposes and stated objectives and whether its development or deployment should proceed." }, { "requirementID": "MANAGE 2.2", "requirementText": "Mechanisms are in place and applied to sustain the value of deployed AI systems." } ] }, "IR 8596: Cybersecurity Framework Profile for Artificial Intelligence (Cyber AI Profile): NIST Community Profile": { "link": "https://csrc.nist.gov/pubs/ir/8596/iprd", "requirements": [ { "requirementID": "GV.OC-01", "requirementText": "The organizational mission is understood and informs cybersecurity risk management" }, { "requirementID": "GV.OC-04", "requirementText": "Critical objectives, capabilities, and services that external stakeholders depend on or expect from the organization are understood and communicated" }, { "requirementID": "PR.AA-05", "requirementText": "Access permissions, entitlements, and authorizations are defined in a policy, managed, enforced, and reviewed, and incorporate the principles of least privilege and separation of duties" } ] }, "SP 800-218A": { "link": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirements": [ { "requirementID": "PO.1.1", "requirementText": "Identify and document all security requirements for the organization’s software development infrastructures and processes, and maintain the requirements over time.\n\nInclude AI model development in the security requirements for software development infrastructure and processes.\n\nIdentify and select appropriate AI model architectures and training techniques in accordance with recommended practices for cybersecurity, privacy, and reproducibility." }, { "requirementID": "PO.1.2", "requirementText": "Identify and document all security requirements for organization-developed software to meet, and maintain the requirements over time.\n\nOrganizational policies should support all current requirements specific to AI model development security for organizationdeveloped software. These requirements should include the areas of AI model development, AI model operations, and data science. Requirements may come from many sources, including laws, regulations, contracts, and standards." }, { "requirementID": "PO.2.3", "requirementText": "Obtain upper management or authorizing official commitment to secure development, and convey that commitment to all with developmentrelated roles and responsibilities.\n\nLeadership should commit to secure development practices involving AI models." } ] } }, "OECD": { "Due Diligence Guidance for Responsible AI": { "link": "https://www.oecd.org/content/dam/oecd/en/publications/reports/2026/02/oecd-due-diligence-guidance-for-responsible-ai_7831bb49/41671712-en.pdf", "requirements": [ { "requirementID": "Step 1.1 – RBC policies", "requirementText": "Devise, adopt and disseminate a combination of policies on RBC issues that articulates the enterprise’s commitments to the principles and standards contained in the OECD AI Principles and the MNE Guidelines. Policies should include plans for implementing due diligence, which will be relevant to the enterprise’s own operations and business relationships in the development and use of AI." }, { "requirementID": "Step 3.2 – Addressing risks directly linked to the enterprise throughout the AI value chain", "requirementText": "Based on the risk prioritisation, develop and implement plans to prevent or mitigate actual or potential adverse impacts directly linked to the enterprise by business relationships (e.g., temporary suspension of the relationship, continuation of the relationship throughout the course of risk mitigation effort, or disengagement).\n\nEnterprises throughout the development and use of AI might be directly linked to adverse impacts caused by (1) other AI actors in the system lifecycle; or (2) business relationships outside of the AI system lifecycle, such as suppliers of AI inputs and users of the AI system.\n\nAppropriate responses to risks associated with business relationships may at times include:\n• continuation of the relationship throughout the course of risk mitigation efforts\n• temporary suspension of the relationship while pursuing ongoing risk mitigation\n• disengagement with the business relationship either after failed attempts at mitigation, or where the enterprise deems mitigation not feasible, or because of the severity of the adverse impact. A decision and subsequent plan to disengage should take into account potential social, environmental and economic adverse impacts and should include meaningful stakeholder engagement. These plans should detail the actions the organisation will take, as well as its expectations of its suppliers, buyers and other business relationships (see Box 2.15). Disengagement actions should be in line with applicable laws, including competition laws." } ] } }, "OpenAI": { "Preparedness Framework": { "link": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "requirements": [ { "requirementID": "Undermining Safeguards", "requirementText": "Undermining Safeguards: ability and propensity for the model to act to undermine safeguards placed on it, including e.g., deception, colluding with oversight models, sabotaging safeguards over time such as by embedding vulnerabilities in safeguards code, etc.\n\nIf a model has High or Critical capabilities in any of the Tracked Categories, require the Safeguards case to be robust to the discovered capability and/or propensity." }, { "requirementID": "Safeguards Against Malicious Users - Robustness", "requirementText": "Malicious users cannot use the model to cause the severe harm because they cannot elicit the necessary capability, such as because the model is modified to refuse to provide assistance to harmful tasks and is robust to jailbreaks that would circumvent those refusals." }, { "requirementID": "Safeguards Against Malicious Users - Trust-based Access", "requirementText": "The actors who gain access to the model are not going to use it in a way that presents an associated risk of severe harm under our threat model." }, { "requirementID": "Safeguards Against a Misaligned Model - Lack of Autonomous Capability", "requirementText": "The model is not capable of carrying out tasks autonomously, including the risk of severe harm." }, { "requirementID": "Safeguards Against a Misaligned Model - System Architecture", "requirementText": "The model can’t take actions that cause harm because it lacks access to output channels or mechanisms to persist sufficiently to execute the harm, due to system design and restricted permissions." } ] }, "Safety Best Practices": { "link": "https://platform.openai.com/docs/guides/safety-best-practices", "requirements": [ { "requirementID": "Prompt engineering", "requirementText": "“Prompt engineering” can help constrain the topic and tone of output text. This reduces the chance of producing undesired content, even if a user tries to produce it. Providing additional context to the model (such as by giving a few high-quality examples of desired behavior prior to the new input) can make it easier to steer model outputs in desired directions." }, { "requirementID": "Constrain user input and limit output tokens", "requirementText": "Limiting the amount of text a user can input into the prompt helps avoid prompt injection. Limiting the number of output tokens helps reduce the chance of misuse.\n\nNarrowing the ranges of inputs or outputs, especially drawn from trusted sources, reduces the extent of misuse possible within an application.\n\nAllowing user inputs through validated dropdown fields (e.g., a list of movies on Wikipedia) can be more secure than allowing open-ended text inputs.\n\nReturning outputs from a validated set of materials on the backend, where possible, can be safer than returning novel generated content (for instance, routing a customer query to the best-matching existing customer support article, rather than attempting to answer the query from-scratch)." }, { "requirementID": "Understand and communicate limitations", "requirementText": "From hallucinating inaccurate information, to offensive outputs, to bias, and much more, language models may not be suitable for every use case without significant modifications. Consider whether the model is fit for your purpose, and evaluate the performance of the API on a wide range of potential inputs in order to identify cases where the API’s performance might drop. Consider your customer base and the range of inputs that they will be using, and ensure their expectations are calibrated appropriately." } ] } }, "OWASP": { "AI Exchange": { "link": "https://owaspai.org/docs/ai_security_overview/", "requirements": [ { "requirementID": "1.1 General governance controls - SEC DEV PROGRAM", "requirementText": "Secure development program: Have processes concerning software development in place to make sure that security is built into your AI system." }, { "requirementID": "1.1 General governance controls - DEV PROGRAM", "requirementText": "Development program: Having a development lifecycle program for AI. Apply general (not just security-oriented) software engineering best practices to AI development." }, { "requirementID": "1.3. Controls to limit the effects of unwanted behaviour - LEAST MODEL PRIVILEGE", "requirementText": "Least model privilege: Minimize what a model can do (trigger actions or access data), to prevent harm in case the model is manipulated, or makes a mistake by itself." }, { "requirementID": "1.3. Controls to limit the effects of unwanted behaviour - MODEL ALIGNMENT", "requirementText": "In the context of Generative AI (e.g., LLMs), alignment refers to the process of ensuring that the model’s behavior and outputs are consistent with human values, intentions, and ethical standards.\n\nControls external to the model to manage model behaviour are:\n\nOVERSIGHT: conventional mechanisms responding to the actual outcome of the model\nLEAST MODEL PRIVILEGE: conventional mechanisms that put boundaries on what the model can affect\nPROMPT INJECTION I/O handling: detection mechanisms on input and output to prevent unwanted behaviour\nThe intent of Model alignment is achieve similar goals by baking it into the model itself, through training and instruction." } ] }, "LLM Top 10": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirements": [ { "requirementID": "LLM01: Prompt Injection - 1", "requirementText": "Provide specific instructions about the model's role, capabilities, and limitations within the system prompt. Enforce strict context adherence, limit responses to specific tasks or topics, and instruct the model to ignore attempts to modify core instructions." }, { "requirementID": "LLM01: Prompt Injection - 4", "requirementText": "Provide the application with its own API tokens for extensible functionality, and handle these functions in code rather than providing them to the model. Restrict the model's access privileges to the minimum necessary for its intended operations.\n" }, { "requirementID": "LLM03: Supply Chain - 2", "requirementText": "Understand and apply the mitigations found in the OWASP Top Ten's \"A06:2021 – Vulnerable and Outdated Components.\" This includes vulnerability scanning, management, and patching components. For development environments with access to sensitive data, apply these controls in those environments, too." }, { "requirementID": "LLM03: Supply Chain - 10", "requirementText": "Encrypt models deployed at AI edge with integrity checks and use vendor attestation APIs to prevent tampered apps and models and terminate applications of unrecognized firmware." }, { "requirementID": "LLM04: Data and Model Poisoning - 4", "requirementText": "Tailor models for different use cases by using specific datasets for fine-tuning. This helps produce more accurate outputs based on defined goals." }, { "requirementID": "LLM06: Excessive Agency - 1", "requirementText": "Limit the extensions that LLM agents are allowed to call to only the minimum necessary. For example, if an LLM-based system does not require the ability to fetch the contents of a URL then such an extension should not be offered to the LLM agent.\n" }, { "requirementID": "LLM06: Excessive Agency - 2", "requirementText": "Limit the functions that are implemented in LLM extensions to the minimum necessary. For example, an extension that accesses a user's mailbox to summarise emails may only require the ability to read emails, so the extension should not contain other functionality such as deleting or sending messages." }, { "requirementID": "LLM06: Excessive Agency - 3", "requirementText": "Avoid the use of open-ended extensions where possible (e.g., run a shell command, fetch a URL, etc.) and use extensions with more granular functionality. For example, an LLM-based app may need to write some output to a file. If this were implemented using an extension torun a shell function then the scope for undesirable actions is very large (any other shell command could be executed). A more secure alternative would be to build a specific filewriting extension that only implements that specific functionality.\n" }, { "requirementID": "LLM06: Excessive Agency - 4", "requirementText": "Limit the permissions that LLM extensions are granted to other systems to the minimum necessary in order to limit the scope of undesirable actions. For example, an LLM agent that uses a product database in order to make purchase recommendations to a customer might only need read access to a 'products' table; it should not have access to other tables, nor the ability to insert, update or delete records. This should be enforced by applying appropriate database permissions for the identity that the LLM extension uses to connect to the database." }, { "requirementID": "LLM06: Excessive Agency - 5", "requirementText": "Track user authorization and security scope to ensure actions taken on behalf of a user are executed on downstream systems in the context of that specific user, and with the minimum privileges necessary. or example, an LLM extension that reads a user's code repo should require the user to authenticate via OAuth and with the minimum scope required.\n" }, { "requirementID": "LLM06: Excessive Agency - 7", "requirementText": "Implement authorization in downstream systems rather than relying on an LLM to decide if an action is allowed or not. Enforce the complete mediation principle so that all requests made to downstream systems via extensions are validated against security policies.\n" }, { "requirementID": "LLM09: Misinformation - 6", "requirementText": "Establish secure coding practices to prevent the integration of vulnerabilities due to incorrect code suggestions.\n" }, { "requirementID": "LLM10: Unbounded Consumption - 6", "requirementText": "Restrict the LLM's access to network resources, internal services, and APIs.\nThis is particularly significant for all common scenarios as it encompasses insider risks and threats. Furthermore, it governs the extent of access the LLM application has to data and resources, thereby serving as a crucial control mechanism to mitigate or prevent side-channel attacks." }, { "requirementID": "LLM10: Unbounded Consumption - 9", "requirementText": "Design the system to degrade gracefully under heavy load, maintaining partial functionality rather than complete failure." }, { "requirementID": "LLM10: Unbounded Consumption - 11", "requirementText": "Train models to detect and mitigate adversarial queries and extraction attempts" }, { "requirementID": "LLM10: Unbounded Consumption - 12", "requirementText": "Build lists of known glitch tokens and scan output before adding it to the model’s context window" } ] }, "OWASP Model Context Protocol (MCP) Top 10": { "link": "https://owasp.org/www-project-mcp-top-10/", "requirements": [ { "requirementID": "MCP01:2025 - Token Mismanagement and Secret Exposure - 2", "requirementText": "Limit Token Lifetime and Scope\n- Issue short-lived, scoped tokens aligned with least privilege principles.\n- Require token renewal for every new MCP session.\n- Bind tokens to the specific agent, tool, or session context." }, { "requirementID": "MCP01:2025 - Token Mismanagement and Secret Exposure - 3", "requirementText": "Enforce Context Isolation\n- Prevent sensitive data persistence in model memory or context windows.\n- Redact or sanitize inputs and outputs before logging.\n- Use ephemeral contexts for operations involving credentials." }, { "requirementID": "MCP02:2025 - Privilege Escalation via Scope Creep - 1", "requirementText": "Least Privilege by Design Define minimal permissions required per agent before deployment. Document intended actions and map them to explicit scopes. Use fine-grained scopes (e.g., repo:write:branch=feature/* rather than repo:write)." }, { "requirementID": "MCP02:2025 - Privilege Escalation via Scope Creep - 3", "requirementText": "Expiry-Based & Just-in-Time (JIT) Access Issue time-limited scopes/tokens for sessions. Require revalidation for long-running or recurring tasks. Use JIT elevation workflows with approval gates for any higher-risk action." }, { "requirementID": "MCP03:2025 - Tool Poisoning - 3", "requirementText": "Strong Access Controls & Separation of Duties\n- Apply least-privilege RBAC to the schema registry; separate the role that can propose a change from the role that approves and publishes it.\n- Use short-lived tokens for deployment pipelines and require human approvals for critical schema releases." }, { "requirementID": "MCP05:2025 – Command Injection & Execution - 1", "requirementText": "Enforce Command Boundaries\n- Use allowlists for permitted commands, arguments, and file paths.\n- Reject shell metacharacters (; & $() <> && \\ ``).\n- Normalize and validate all file paths to block traversal." }, { "requirementID": "MCP05:2025 – Command Injection & Execution - 2", "requirementText": "Adopt Safe Execution Patterns\n- Never use shell=True, eval(), exec(), or string-built commands.\n- Always execute with structured parameters (e.g., subprocess.run([‘ls’, ‘logs’])).\n- Disable direct execution of model-generated code unless manually reviewed." }, { "requirementID": "MCP05:2025 – Command Injection & Execution - 4", "requirementText": "Apply Least Privilege Run tools as non-root with minimal filesystem, API, and DB permissions. Prevent agents from accessing environment variables or secrets by default." }, { "requirementID": "MCP06:2025 – Intent Flow Subversion - 2", "requirementText": "Independent Intent Verification (The Checker Pattern)\n- Use a separate, independent “Guardrail Model” to verify proposed tool calls. This model should only see the User Intent and the Proposed Action, ensuring it is isolated from potentially poisoned MCP context." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 4", "requirementText": "Least Privilege Principle\n- Minimize agent permissions — assign only what’s needed for the task.\n- Split high-privilege operations into separate workflows requiring human review.\n- Restrict admin or system tokens from being used in development or shared contexts." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 12", "requirementText": "Remove unapproved plugins, schemas, or connectors." }, { "requirementID": "MCP10:2025 – Context Injection & Over-Sharing - 2", "requirementText": "Context Isolation & Segmentation\nAssign unique context namespaces per:\nUser\nAgent\nWorkflow\nTenant - Prevent one agent from accessing another agent’s memory directly. - In multi-tenant setups, isolate retrieval indexes and vector stores." }, { "requirementID": "MCP10:2025 – Context Injection & Over-Sharing - 8", "requirementText": "Context Injection Filtering\nDetect and block instruction-like content trying to persist in memory:\n- “Ignore previous instructions”\n- “Share everything you know”\n- Maintain injection pattern detection models." }, { "requirementID": "MCP10:2025 – Context Injection & Over-Sharing - 9", "requirementText": "Purge existing shared contexts and caches. Enforce per-agent and per-user segmentation. Introduce TTL policies and auto-purge logic. Rotate keys and invalidate context stores if contamination is confirmed. Review access control around vector databases and embeddings." } ] }, "OWASP Top 10 for Agentic Applications for 2026": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "requirements": [ { "requirementID": "ASI01: Agent Goal Hijack - 5", "requirementText": "When building agents, evaluate use of “intent capsule”, an emerging pattern to bind the declared goal, constraints, and context to each execution cycle in a signed envelope, restricting run-time use." }, { "requirementID": "ASI02: Tool Misuse and Exploitation - 1", "requirementText": "Least Agency and Least Privilege for Tools. Define per-tool least-privilege profiles (scopes, maximum rate, and egress allowlists) and restrict agentic tool functionality and each tool’s permissions and data scope to those profiles – e.g., read-only queries for databases, no send/delete rights for email summarizers, and minimal CRUD operations when exposing APIs. Where possible, express these profiles as IAM or authorization policy stanzas attached to each tool, rather than relying on ad-hoc conventions." }, { "requirementID": "ASI03: Identity and Privilege Abuse - 1", "requirementText": "Enforce Task-Scoped, Time-Bound Permissions: Issue short-lived, narrowly scoped tokens per task and cap rights with permission boundaries - using per-agent identities and short-lived credentials (e.g., mTLS certificates or scoped tokens) - to limit blast radius, block delegated-abuse and maintenance-window attacks, and mitigate un-scoped inheritance, orphaned privileges, and reflection-loop elevation." }, { "requirementID": "ASI03: Identity and Privilege Abuse - 2", "requirementText": "Isolate Agent Identities and Contexts: Run per-session sandboxes with separated permissions and memory, wiping state between tasks to prevent Memory-Based Escalation and reduce Cross- Repository Data Exfiltration." }, { "requirementID": "ASI03: Identity and Privilege Abuse - 3", "requirementText": "Mandate Per-Action Authorization: Re-verify each privileged step with a centralized policy engine that checks external data, stopping Cross-Agent Trust Exploitation and Reflection Loop Elevation." }, { "requirementID": "ASI03: Identity and Privilege Abuse - 5", "requirementText": "Define Intent: Bind OAuth tokens to a signed intent that includes subject, audience, purpose, and session. Reject any token use where the bound intent doesn’t match the current request." }, { "requirementID": "ASI03: Identity and Privilege Abuse - 6", "requirementText": "Evaluate Agentic Identity Management Platforms. Major platforms integrate agents into their identity and access management systems, treating them as managed non-human identities with scoped credentials, audit trails, and lifecycle controls. Examples include Microsoft Entra, AWS Bedrock Agents, Salesforce Agentforce, Workday’s Agentic System of Record (ASOR) model, and similar emerging patterns in Google Vertex AI." }, { "requirementID": "ASI03: Identity and Privilege Abuse - 7", "requirementText": "Bind permissions to subject, resource, purpose, and duration. Require re-authentication on context switch. Prevent privilege inheritance across agents unless the original intent is re-validated. Include automated revocation on idle or anomaly." }, { "requirementID": "ASI03: Identity and Privilege Abuse - 8", "requirementText": "Detect Delegated and Transitive Permissions: Monitor when an agent gains new permissions indirectly through delegation chains. Flag cases where a low-privilege agent inherits or is handed higher-privilege scopes during multi-agent workflows." }, { "requirementID": "ASI04: Agentic Supply Chain Vulnerabilities - 9", "requirementText": "Zero-trust security model in application design: design system with security fault tolerance that assumes failure or exploitation of LLM or agentic function components." }, { "requirementID": "ASI06: Memory & Context Poisoning - 6", "requirementText": "Prevent automatic re-ingestion of an agent’s own generated outputs into trusted memory to avoid self-reinforcing contamination or “bootstrap poisoning.”" }, { "requirementID": "ASI06: Memory & Context Poisoning - 8", "requirementText": "Expire unverified memory to limit poison persistence." }, { "requirementID": "ASI06: Memory & Context Poisoning - 9", "requirementText": "Weight retrieval by trust and tenancy: Require two factors to surface high-impact memory (e.g., provenance score plus human-verified tag) and decay low-trust entries over time." }, { "requirementID": "ASI07: Insecure Inter-Agent Communication - 9", "requirementText": "Typed contracts and schema validation: Use versioned, typed message schemas with explicit per-message audiences. Reject messages that fail validation or attempt schema down-conversion without declared compatibility. Typed contracts help with structure, but semantic divergence across agents remains an inherent challenge; mitigations therefore focus on integrity, provenance, and controlled communication patterns rather than attempting full semantic alignment." }, { "requirementID": "ASI08: Cascading Failures - 1", "requirementText": "Zero-trust model in application design: design system with fault tolerance that assumes availability failure of LLM:2025, agentic function components and external sources." }, { "requirementID": "ASI08: Cascading Failures - 4", "requirementText": "Independent policy enforcement: Separate planning and execution via an external policy engine to prevent corrupt planning from triggering harmful actions." } ] } }, "Personal Data Protection Commission Singapore (PDPC)": { "Model Artificial Intelligence Governance Framework Second Edition": { "link": "https://www.pdpc.gov.sg/-/media/files/pdpc/pdf-files/resource-for-organisation/ai/sgmodelaigovframework2.pdf", "requirements": [ { "requirementID": "Repeatability - c)", "requirementText": "Ensuring exception handling is in line with organisations’ policies;" } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirements": [ { "requirementID": "6.3", "requirementText": "An Entity should ensure that its strategy provides a business case, addresses information and communication technology requirements, information security, and operational risk management including business continuity, disaster recovery, and resiliency framework." }, { "requirementID": "6.4", "requirementText": "An Entity's Al strategy should define an implementation plan and architectural roadmap which covers the target IT environment, the transition from the current environment to the target environment and the operating model, including any organizational change or additional skillsets that may be necessary." }, { "requirementID": "6.5", "requirementText": "An Entity must allocate sufficient resources to handle any Al projects and ongoing business needs." }, { "requirementID": "9.2", "requirementText": "An Entity must evaluate the use of the Al System in critical organizational processes." }, { "requirementID": "12.9", "requirementText": "An Entity must also have a contingency plan in the event that the arrangement with the outsourcing service provider is suddenly terminated." }, { "requirementID": "13.7.5", "requirementText": "An Entity must ensure an Al System is subject to built-in operational constraints that cannot be overridden by the system itself." }, { "requirementID": "16.2", "requirementText": "The implementation of aspects referred to in clause (16.1) must be proportionate to the size of the Provider's organization and use of High-Risk Al." }, { "requirementID": "17.3", "requirementText": "An Entity's TRiSM program must ensure that Al Providers adhere to their program which includes:\n- Utilizing defined Al Model management processes.\n- Building Al Models with defined controls against security breaches." }, { "requirementID": "17.4", "requirementText": "An Entity must examine any Al Model's attack surface prior to deployment, and address any security findings." }, { "requirementID": "17.5", "requirementText": "An Entity must ensure that its Al model is protected from integrity related attacks to prevent model manipulation." }, { "requirementID": "17.6", "requirementText": "An Entity must ensure that the Al model is protected against query attacks that may lead to model manipulation or theft." }, { "requirementID": "17.7", "requirementText": "An Entity must ensure that the Al model is protected against prompt injections that may lead to data poisoning or data drift." } ] } }, "SANS": { "Critical AI Security Guidelines": { "link": "https://sansorg.egnyte.com/dl/bvkYQxrW8QMj", "requirements": [ { "requirementID": "3.3 Implement Access Controls Outside of the Model", "requirementText": "Attempting to implement these types of controls within the model is error prone and can often be easily subverted. Instead, consider a RAG-style approach with access control lists (ACLs) applied in the vector retrieval system from which responses are generated. This eliminates the need to attempt to implement these guardrails in the LLM. This approach also has the not-so-subtle benefit of limiting the likelihood of so-called hallucinations in the responses from the LLM. In addition, organizations should pay close attention to the use of function calling, especially in agentic AI systems. If not properly scoped, function calls may allow models to invoke external tools or actions beyond their intended purpose. Limit access to critical functions and monitor usage." }, { "requirementID": "4.3 Sanitize, Validate, and Filter LLM Outputs/Responses", "requirementText": "Adversaries employ prompt injection to get the LLM application to do or say something it should not. Although trying to prevent or detect the attempted inject should be considered necessary, the complexity and nuance of LLM applications make it obvious that merely controlling the input should not be considered sufficient. Additionally, prompt injection primarily focuses on intentional abuse or misuse, yet inputs could still result in undesirable LLM application responses or behaviors. Much as validation and filtering of inputs proves vital, so too is properly handling and assessing outputs. Keep in mind that, like inputs, multiple layers and levels of output might exist in a complex LLM application, such as one that employs web search, function calling, tool use, or downstream LLMs. Output should not be construed to refer only to what would be presented to an end user" }, { "requirementID": "4.4 Employ the Principle of Focused Functionality (and Agency)", "requirementText": "Models continuously evolve, acquiring tremendous new capabilities and achieving previously unthinkable milestones. Despite this, LLM applications should offer as limited functionality as is acceptable. Since the 1990s, Bruce Schneier has been offering some version of the mantra, “The worst enemy of security is complexity.” In designing agents, it is advisable to explicitly define and limit the functions and tools (code interpreters, web search, and other external APIs) the agent requires access to in order to fulfill its tasks. Avoid assigning multiple tools to an agent and apply the principle of least privilege." } ] } }, "SDAIA (Saudi Arabia)": { "AI Adoption Framework": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Files/AIAdoptionFramework.pdf", "requirements": [ { "requirementID": "5.1.2 Privacy and Safety - Embedding Cybersecurity and Privacy", "requirementText": "Designing modern intelligent systems requires a “Security & Privacy by Design” approach, integrating protection requirements into every development stage, from analysis to deployment. This includes data encryption, access management, and multi-factor authentication to safeguard the system from vulnerabilities and attacks before official launch." } ] }, "AI Ethics Principles": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Documents/ai-principles.pdf", "requirements": [ { "requirementID": "Principle 2 – Privacy & Security - Plan and Design - 1", "requirementText": "The planning and design of the AI system and its associated algorithm must be configured and modelled in a manner such that there is respect for the protection of the privacy of individuals, personal data is not misused and exploited, and the decision criteria of the automated technology is not based on personally identifying characteristics or information." }, { "requirementID": "Principle 2 – Privacy & Security - Plan and Design - 2", "requirementText": "The use of personal information should be limited only to that which is necessary for the proper functioning of the system. The design of AI systems resulting in the profiling of individuals or communities may only occur if approved by Chief Compliance and Ethics Officer, Compliance Officer or in compliance with a code of ethics and conduct developed by a national regulatory authority for the specific sector or industry" }, { "requirementID": "Principle 2 – Privacy & Security - Plan and Design - 3", "requirementText": "The security and protection blueprint of the AI system, including the data to be processed and the algorithm to be used, should be aligned to best practices to be able to withstand cyberattacks and data breach attempts." }, { "requirementID": "Principle 2 – Privacy & Security - Plan and Design - 4", "requirementText": "Privacy and security legal frameworks and standards should be followed and customized for the particular use case or organization." }, { "requirementID": "Principle 2 – Privacy & Security - Plan and Design - 5", "requirementText": "An important aspect of privacy and security is data architecture; consequently, data classification and profiling should be planned to define the levels of protection and usage of personal data." }, { "requirementID": "Principle 2 – Privacy & Security - Build and Validate - 1", "requirementText": "Privacy and security by design should be implemented while building the AI system. The security mechanisms should include the protection of various architectural dimensions of an AI model from malicious attacks. The structure and modules of the AI system should be protected from unauthorized modification or damage to any of its components." }, { "requirementID": "Principle 2 – Privacy & Security - Build and Validate - 2", "requirementText": "The AI system should be secure to ensure and maintain the integrity of the information it processes. This ensures that the system remains continuously functional and accessible to authorized users. It is crucial that the system safeguards confidential and private information, even under hostile or adversarial conditions. Furthermore, appropriate measures should be in place to ensure that AI systems with automated decision-making capabilities uphold the necessary data privacy and security standards." }, { "requirementID": "Principle 5 – Reliability & Safety - Plan and Design - 1", "requirementText": "Designing and developing an AI system that can withstand the uncertainty, instability, and volatility that it might encounter is crucial." }, { "requirementID": "Principle 5 – Reliability & Safety - Plan and Design - 3", "requirementText": "Establishing a set of standards and protocols for assessing the reliability of an AI system is necessary to secure the safety of the system’s algorithm and data output. It is essential to keep a sustainable technical outlay and outcomes generated from the system to maintain the public’s trust and confidence in the AI system." }, { "requirementID": "Principle 6 – Transparency & Explainability - Build and Validate - 1", "requirementText": "Transparency in AI is thought about from two perspectives, the first is the process behind it (the design and implementation practices that lead to an algorithmically supported outcome) and the second is in terms of its product (the content and justification of that outcome). Algorithms should be developed in a transparent way to ensure that input transparency is evident and explainable to the end-users of the AI system to be able to provide evidence and information on the data used to process the decisions that have been processed." } ] }, "Generative AI Guidelines": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Files/GenerativeAIPublicEN.pdf", "requirements": [ { "requirementID": "4.5 Privacy & Security - 4", "requirementText": "Privacy and security by design should be implemented while building the AI system. The security mechanisms should include the protection of various architectural dimensions of an AI model from malicious attacks. The structure and modules of the AI system should be protected from unauthorized modification or damage to any of its components." } ] } }, "Smart Dubai (UAE)": { "AI Ethics Principles & Guidelines": { "link": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirements": [ { "requirementID": "1.1.1.2", "requirementText": "AI developer organisations and AI operator organisations should refrain from training AI systems on data that is not likely to be representative of the affected AI subjects, or is not likely to be accurate, whether that be due to age, omission, method of collection, or other factors." }, { "requirementID": "1.2.2.1", "requirementText": "AI operator organisations should only use AI systems that are backed by respected and evidence-based academic research, and AI developer organisations should base their development on such research." } ] } }, "U.S. Department of Health & Human Services": { "Trustworthy AI (TAI) Playbook: Executive Summary": { "link": "https://www.hhs.gov/sites/default/files/hhs-trustworthy-ai-playbook-executive-summary.pdf", "requirements": [ { "requirementID": "Robust / Reliable", "requirementText": "AI systems should have the ability to learn from humans and other systems and produce accurate and reliable outputs consistent with the original design" } ] } }, "World Economic Forum": { "Presidio AI Framework: Towards Safe Generative AI Models": { "link": "https://www3.weforum.org/docs/WEF_Presidio_AI%20Framework_2024.pdf", "requirements": [ { "requirementID": "Expanded AI life cycle", "requirementText": "The expanded AI life cycle encompasses risks and guardrails with varying safety benefits and challenges throughout each phase.\nThe expanded AI life cycle synthesizes elements from data management, foundation model design and development, release access, use of generative capabilities and adaptation to a use case." }, { "requirementID": "Guardrails across the expanded AI life cycle", "requirementText": "Implementation of known and novel guardrails is necessary for safe systems to ensure technical quality, consistency and control." } ] } } }, "Principle 3": { "CEN/CENELEC": { "prEN 40000-1-1": { "link": "https://genorma.com/en/standards/pren-40000-1-1", "requirements": [ { "requirementID": "acceptable risk", "requirementText": "the level of risk deemed acceptable for intended and reasonably foreseeable use, considering state of the art and safety." }, { "requirementID": "residual cybersecurity risk", "requirementText": "risk remaining after treatment." } ] }, "prEN 40000-1-2: Cybersecurity requirements for products with digital elements - Part 1-2: Principles for cyber resilience": { "link": "https://genorma.com/en/standards/pren-40000-1-2", "requirements": [ { "requirementID": "5.2", "requirementText": "Risk Based Approach to Cybersecurity" }, { "requirementID": "6.3", "requirementText": "Risk acceptance criteria and risk management methodology" }, { "requirementID": "6.4", "requirementText": "Risk assessment" }, { "requirementID": "6.6", "requirementText": "Risk Communication" }, { "requirementID": "6.7", "requirementText": "Risk monitoring " } ] } }, "Central Bank of the UAE": { "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E": { "link": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirements": [ { "requirementID": "2. Governance and Accountability - c", "requirementText": "Regular reporting should be required by and provided to Senior Management and Boards of LFIs, covering performance and risk." }, { "requirementID": "2. Governance and Accountability - d", "requirementText": "Governance structures should facilitate informed decision-making, enable the identification and mitigation of risks and ensure that AI and ML systems and applications are aligned with the institution’s risk appetite and legal obligations. AI-related risks should be incorporated into the institution’s governance framework in a cohesive and consolidated manner, including with specific adaptable roles and responsibilities for the Audit and Risk Committee, Risk Management, Internal Audit, and IT. " }, { "requirementID": "2. Governance and Accountability - e", "requirementText": "Boards and senior management should ensure that risk committees and control functions (e.g., compliance, internal audit and risk management) understand AI-driven processes and can challenge outcomes where appropriate, with the adoption, deployment and use of AI being an integral part of the Risk Management framework." }, { "requirementID": "8. Integration with Existing Frameworks - a", "requirementText": "Whilst the focus of this guidance is Consumer Protection, risk management with respect to AI and ML activities should be embedded within the institution’s enterprise-wide risk management framework, including conduct risk, credit risk, operational risk and cybersecurity risk. AI risk assessments should not operate in isolation but should inform and be informed by the institution’s overall risk appetite and controls." }, { "requirementID": "8. Integration with Existing Frameworks - d", "requirementText": "LFIs should create processes to rate the risk of each AI system/application/technology they deploy or use, to enable appropriate risk assessment, monitoring and management of the AI whilst deployed in use and developing, which may be influenced by data quality and sensitivity, capability of the AI, controls in place, impact of the AI and dependence on the AI and/or third parties in the use of the AI." } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirements": [ { "requirementID": "2.2.3 - Exposure of Sensitive Information", "requirementText": "Minimize risk by not sharing sensitive data with AI models, especially if the AI models are hosted in an environment hosted or controlled by external parties, such as public cloud infrastructure." }, { "requirementID": "4.1.2 - Understand the correctness of AI system results to support continued safe operation of systems in an OT environment. ", "requirementText": "It is vital for critical infrastructure owners and operators to understand the states where an AI system can fail to produce accurate and reliable results. This understanding includes expectations for false positives and false negatives in the system’s performance, and how the false positives compare to the base rate of true positives." }, { "requirementID": "4.2.1 - Establish failsafe mechanisms that enable AI systems to fail gracefully without disrupting critical operations.", "requirementText": "Incorporate new AI system failure states, including how to bypass or replace an AI system, into existing functional safety and incident response processes. Integrating an AI system into existing OT networks inevitably generates new failure states for the overall critical infrastructure system. Therefore, operators responsible for revising the existing functional safety and incident response processes should incorporate these new failure states as they are critical to ensuring safe operation of these systems." }, { "requirementID": "4.2.3 - Incorporate AI considerations into the cybersecurity incident response plan.", "requirementText": "Despite organizations’ best efforts at mitigation, risk cannot be reduced to zero; incidents are inevitable. To account for this, critical infrastructure owners and operators should update their incident response plans and functional safety procedures with steps for responding to malicious activity directed against an AI system and AI system failure." } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "A&A-06", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain a risk-based corrective action plan to remediate audit findings, regularly review and report remediation status to relevant stakeholders." }, { "requirementID": "AIS-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for application security. Review and update the policies and procedures at least annually or after significant system changes." }, { "requirementID": "BCR-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain business continuity management and operational resilience policies and procedures. Review and update the policies and procedures at least annually, or when significant changes occur that could impact risk exposure." }, { "requirementID": "BCR-02", "requirementText": "Determine the impact of business disruptions and risks to establish criteria for developing business continuity and operational resilience strategies and capabilities. Review and update the risk assessment and impact analysis at least annually or upon significant changes." }, { "requirementID": "CEK-06", "requirementText": "Manage and adopt changes to cryptography-, encryption-, and key management-related systems (including policies and procedures) that fully account for downstream effects of proposed changes, including residual risk, cost, and benefits analysis." }, { "requirementID": "CEK-07", "requirementText": "Establish and maintain an encryption and key management risk program that includes provisions for risk assessment, risk treatment, risk context, monitoring, and feedback." }, { "requirementID": "CEK-20", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to assess the risk to operational continuity versus the risk of the keying material and the information it protects being exposed if control of the keying material is lost, which include provisions for legal and regulatory requirements." }, { "requirementID": "DCS-05", "requirementText": "Classify and document the physical, and logical assets (e.g., applications) based on the organizational business risk. Review and update the assets’ classification at least annually or upon significant changes." }, { "requirementID": "DSP-09", "requirementText": "Conduct a Data Protection Impact Assessment (DPIA) to evaluate the origin, nature, particularity and severity of the risks upon the processing of personal data, according to any applicable laws, regulations and industry best practices." }, { "requirementID": "DSP-21", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to prevent data poisoning in AI models and continuously detect such." }, { "requirementID": "GRC-02", "requirementText": "Establish and maintain a formal, documented, and leadership-sponsored AI Risk Management (AIRM) program that includes policies and procedures for identification, evaluation, ownership, treatment, and acceptance of risks." }, { "requirementID": "GRC-03", "requirementText": "Review all relevant organizational policies and associated procedures at least annually or when a substantial change occurs within the organization." }, { "requirementID": "GRC-09", "requirementText": "Define, document and enforce policies and procedures on the acceptable use of AI services offered by the organization. Ensure effectiveness by continuous risk assessments, reviews and human oversight." }, { "requirementID": "GRC-10", "requirementText": "Establish, document, and communicate to all relevant stakeholders an AI Impact Assessment process and its criteria to regularly evaluate the ethical, societal, operational, legal, and security impacts of the AI system throughout its lifecycle." }, { "requirementID": "IAM-08", "requirementText": "Review and revalidate user access for least privilege and separation of duties with a frequency that is commensurated with organizational risk tolerance and at least annually, or upon significant changes." }, { "requirementID": "I&S-08", "requirementText": "Identify and document high-risk environments." }, { "requirementID": "MDS-01", "requirementText": "Define, implement, and evaluate policies, procedures, and technical measures that ensure the security of the Training Pipeline. Regularly review and update policies, procedures and technical measures to address new security threats and best practices." }, { "requirementID": "MDS-06", "requirementText": "Define, implement, and evaluate processes and technical measures to assess adversarial threats specific to each AI model." }, { "requirementID": "MDS-11", "requirementText": "Perform a risk-based evaluation of the model and model serving infrastructure for model failure. Define and implement measures to mitigate model and model serving infrastructure failures, and regularly evaluate throughout the AI system's lifecycle." }, { "requirementID": "MDS-12", "requirementText": "Establish a process to evaluate risk associated with open models. Periodically review these risk factors, and implement a process to monitor and mitigate any determined vulnerabilities." }, { "requirementID": "STA-15", "requirementText": "Define and implement a process for conducting security assessments periodically for all organizations within the supply chain." }, { "requirementID": "TVM-08", "requirementText": "Use a risk-based model for effective prioritization of vulnerability remediation using an industry recognized framework." }, { "requirementID": "TVM-11", "requirementText": "Define and implement processes, procedures and technical measures to apply guardrails to the AI system. Continuously evaluate guardrails for changes in regulatory requirements and risk scenarios." }, { "requirementID": "TVM-12", "requirementText": "Define implement and evaluate threat analysis process and procedures to identify, assess and review the threat landscape for Cloud and AI systems. Build threat models according to industry best practices to inform the risk mitigation strategy." }, { "requirementID": "TVM-13", "requirementText": "Use a risk-based method for the prioritization and mitigation of threats, leveraging an industry-recognized framework to guide threat decision-making and protection measures." } ] } }, "CoSAI": { "AI Incident Response Framework": { "link": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirements": [ { "requirementID": "3.3.1. Preparation Phase - Risk Assessment & Threat Modeling", "requirementText": "• Identify critical assets in each architecture pattern\n• Map ATLAS threat vectors to components\n• Assess likelihood and impact\n• Prioritize security controls\n• Document risk thresholds" } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "2.1.2", "requirementText": "The entity shall identify, assess, and mitigate unique cyber security risks associated with AI/ML systems in a systematic and ongoing manner." }, { "requirementID": "2.2.4", "requirementText": "The entity shall implement a systematic process to identify, assess, and remediate vulnerabilities in AI/ML systems to enhance their resilience against cyber security threats." }, { "requirementID": "2.3.3", "requirementText": "The entity shall ensure the security of the AI/ML inference process by protecting against potential threats and maintaining the integrity and confidentiality of inference data." }, { "requirementID": "3.1.2 Cyber Risk Management for AI/ML - 1", "requirementText": "The entity should have a formal cyber security risk management process that specifically addresses the unique cyber security risks associated with AI/ML." }, { "requirementID": "3.1.2 Cyber Risk Management for AI/ML - 2", "requirementText": "The AI/ML cyber security risk management process should be integrated into the entity's broader risk management program, ensuring that cyber security risks associated with AI/ML systems are classified, prioritized, and considered as part of the organization's overall risk profile." }, { "requirementID": "3.1.2 Cyber Risk Management for AI/ML - 3", "requirementText": "The entity should conduct cyber security risk assessments at regular intervals and at key stages in the AI/ML lifecycle, such as during model design, data collection, training, and deployment." }, { "requirementID": "3.1.2 Cyber Risk Management for AI/ML - 4", "requirementText": "The entity should assess and mitigate cyber security risks related to cloud-based AI solutions, particularly those that utilize client data under the End User License Agreement (EULA) for unsupervised learning in the background." }, { "requirementID": "3.1.2 Cyber Risk Management for AI/ML - 5", "requirementText": "The entity should regularly review and update the AI/ML cyber security risk management process to address new threats, vulnerabilities, and risk mitigation strategies." }, { "requirementID": "3.1.4 Change Management and Reporting - 2", "requirementText": "This process should consider the potential cyber security impacts of changes, including changes that could affect the confidentiality, integrity, or availability of AI/ML systems or the data they process." }, { "requirementID": "3.2.2 Security Configuration Management - 2", "requirementText": "The entity should periodically review and update AI/ML secure configuration baselines to ensure continued effectiveness against evolving threats and changes in system architecture or technology." }, { "requirementID": "3.3.1 Security by Design for AI/ML Models - 2", "requirementText": "The entity should perform threat modeling during the design and development of AI/ML models to identify potential cyber security threats and design appropriate mitigating controls." }, { "requirementID": "3.6.2 Incident Reporting and Management for AI/ML - 6", "requirementText": "The entity should conduct periodic reviews and updates to the automated response rules based on lessons learned from exercises and actual incidents to ensure continuous improvement and alignment with emerging threats." }, { "requirementID": "3.6.3 Digital Forensics for AI/ML Security Incidents - 3", "requirementText": "The entity should incorporate the forensic insights into the post-incident review process, cyber security strategy and risk management, for enhancing the overall resilience of its AI/ML systems." } ] } }, "ENISA": { "Multilayer Framework for Good Cybersecurity Practices for AI": { "link": "https://www.enisa.europa.eu/sites/default/files/publications/Multilayer%20Framework%20for%20Good%20Cybersecurity%20Practices%20for%20AI.pdf", "requirements": [ { "requirementID": "Networking 8", "requirementText": "Do you impose dynamic risk assessment to be conducted by the AI stakeholders?" } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.1.3-1", "requirementText": "Developers and System Operators shall analyse threats and manage security risks to their systems. Threat modelling should include regular reviews and updates and address AI-specific attacks, such as data poisoning, model inversion, and membership inference." }, { "requirementID": "Provision 5.1.3-1.1", "requirementText": "The threat modelling and risk management process shall be conducted to address any security risks that arise when a new setting or configuration option is implemented or updated at any stage of the AI lifecycle." }, { "requirementID": "Provision 5.1.3-1.2", "requirementText": "Developers shall manage the security risks associated with AI models that provide superfluous functionalities, where increased functionality leads to increased risk. For example, where a multi-modal model is being used but only single modality is used for system function." }, { "requirementID": "Provision 5.1.3-1.3", "requirementText": "System Operators shall apply controls to risks identified through the analysis based on a range of considerations, including the cost of implementation in line with their corporate risk tolerance." }, { "requirementID": "Provision 5.1.3-2", "requirementText": "Where AI security threats are identified that cannot be resolved by Developers, this shall be communicated to System Operators so they can threat model their systems. System Operators shall communicate this information to End-users, so they are made aware of these threats. This communication should include detailed descriptions of the risks, potential impacts, and recommended actions to address or monitor these threats." }, { "requirementID": "Provision 5.1.3-3", "requirementText": "Where an external entity has responsibility for AI security risks identified within an organizations infrastructure, System Operators should attain assurance that these parties are able to address such risks" }, { "requirementID": "Provision 5.1.3-4", "requirementText": "Developers and System Operators should continuously monitor and review their system infrastructure according to risk appetite. It is important to recognize that a higher level of risk will remain in AI systems despite the application of controls to mitigate against them." } ] }, "SAI 002 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_gr/SAI/001_099/002/01.01.01_60/gr_SAI002v010101p.pdf", "requirements": [ { "requirementID": "6.1.3 Supply chain security - 1", "requirementText": "Understanding the risks associated with the supply chain, particularly for high-value components such as datasets. This includes understanding the security posture of the suppliers." }, { "requirementID": "6.5 - Following standard cybersecurity supply chain guidance", "requirementText": "Data, models and the roles and risks associated with them can be understood and assessed in the same way as any other component of a system." } ] }, "TR 104 048 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_tr/104000_104099/104048/01.01.01_60/tr_104048v010101p.pdf", "requirements": [ { "requirementID": "6.5 Analysis - Hash checks", "requirementText": "Existing cryptographic mechanisms can be used for protecting the integrity of data in an efficient way. For verification of data integrity there is a trade-off between efficiency and security, which should be balanced according to the risk level of the application." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.1.3-1", "requirementText": "\"Developers and System Operators shall analyse threats and manage security risks to their systems. Threat modelling should include regular reviews and updates and address AI-specific attacks, such as data poisoning, model inversion, and membership inference.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nAI systems face unique threats, such as data poisoning, model inversion, and membership inference attacks, which traditional threat models cannot account for. New threats will emerge that will need to be incorporated in threat modelling and risk management.\n\nExample Measures/Controls:\nPerform Threat Modelling including AI threats: Apply threat modelling that captures potential impacts on stakeholders including both AI and traditional cyberattacks. Document each identified threat in detail, outlining the likelihood and severity of potential impacts to the AI model and the broader system and list mitigations using standardized OWASP or MITRE controls. Both OWASP AI Exchange [i.10] or MITRE ATLAS [i.8] provide threat taxonomies and related mitigations which both types of attacks and they can be used in threat modelling. If the AI system processes or was built on personal data ICO's guidance on AI and security is useful to consult for regulatory compliance." }, { "requirementID": "Provision 5.1.3-1.1", "requirementText": "\"The threat modelling and risk management process shall be conducted to address any security risks that arise when a new setting or configuration option is implemented or updated at any stage of the AI lifecycle.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nFailure to conduct threat modelling and risk management when implementing or updating settings or configurations during the AI lifecycle can lead to unmitigated security vulnerabilities, such as configuration errors or unanticipated attack vectors, increasing the risk of exploitation and system compromise.\n\nExample Measures/Controls:\nConduct Threat Modelling for Configuration Changes: Perform threat modelling whenever settings or configurations are implemented or updated to identify and mitigate security risks throughout the AI lifecycle. " }, { "requirementID": "Provision 5.1.3-1.2", "requirementText": "\"Developers shall manage the security risks associated with AI models that provide superfluous functionalities, where increased functionality leads to increased risk. For example, where a multi-modal model is being used but only single modality is used for system function.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nAllowing AI models to retain superfluous functionalities that are not required for the system's purpose can introduce unnecessary security risks, such as expanded attack surfaces, increased vulnerability to exploitation, and potential misuse of unused features, compromising the overall security of the system.\n\nExample Measures/Controls 1:\nRestrict Superfluous Functionalities: Limit AI model functionalities to those essential for the system's purpose to reduce the attack surface and minimize security risks associated with unused features.\n\nExample Measures/Controls 2:\nIntegrate Threat Modelling with AI Governance: Require completed threat models for governance approval at critical stages of the AI lifecycle, ensuring documented risk understanding and mitigation before deployment providing support and guidance and ensuring cross-discipline input (ethics, privacy, legal, etc.) to threat modelling." }, { "requirementID": "Provision 5.1.3-1.3", "requirementText": "\"System Operators shall apply controls to risks identified through the analysis based on a range of considerations, including the cost of implementation in line with their corporate risk tolerance.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWhen risk tolerance is not clearly defined in the context of AI-specific risks such as data poisoning or model misuse, this could result into Inadequately prioritized controls leading to breaches, operational disruptions, or unethical decision-making.\n\nExample Measures/Controls:\nDevelop a Prioritization Framework for AI Risk Controls: Use an AI-specific risk-scoring system to prioritize mitigations and controls based on the impact of threats, likelihood of occurrence, and in alignment with organizational risk tolerance. This should account for regulatory risk, including data protection, and cover AI-specific vulnerabilities, such as adversarial manipulation, model drift, and bias." }, { "requirementID": "Provision 5.1.3-2", "requirementText": "\"Where AI security threats are identified that cannot be resolved by Developers, this shall be communicated to System Operators so they can threat model their systems. System Operators shall communicate this information to End-users, so they are made aware of these threats. This communication should include detailed descriptions of the risks, potential impacts, and recommended actions to address or monitor these threats.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout clear communication on unresolved risks, System Operators and End-users can lack awareness, limiting their ability to apply safeguards effectively.\n\nExample Measures/Controls:\nDocument and Communicate Identified Unresolved Risks: Ensure clear documentation and timely communication of any unresolved threats to all relevant stakeholders." }, { "requirementID": "Provision 5.1.3-3", "requirementText": "\"Where an external entity has responsibility for AI security risks identified within an organizations infrastructure, System Operators should attain assurance that these parties are able to address such risks.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nReliance on third parties without adequate verification could expose the AI system to unmanaged vulnerabilities.\n\nExample Measures/Controls:\nConduct AI-Specific Security Assessments for Third Parties: Ensure third-party components and vendors undergo security assessments that specifically address AI-related risks and adherence with ETSI TS 104 223 [i.1]. " }, { "requirementID": "Provision 5.1.3-4", "requirementText": "\"Developers and System Operators should continuously monitor and review their system infrastructure according to risk appetite. It is important to recognize that a higher level of risk will remain in AI systems despite the application of controls to mitigate against them.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nResidual risk can be exploited by malicious actors, especially as evolving threats introduce new vulnerabilities or amplify existing ones, leading to potential breaches, disruptions, or compromised AI integrity.\n\nExample Measures/Controls:\nEstablish Continuous AI Risk Monitoring Controls: Implement a regular review processes of AI developments to determine whether emerging vulnerabilities, improved mitigation techniques, or advancements in AI models necessitates updates to the risk assessment controls." } ] } }, "EU ": { "EU AI Act": { "link": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirements": [ { "requirementID": "9.1 Risk Management System", "requirementText": "A risk management system shall be established, implemented, documented and maintained in relation to high-risk AI systems." }, { "requirementID": "9.2 Risk Management System", "requirementText": "The risk management system shall be understood as a continuous iterative process planned and run throughout the entire lifecycle of a high-risk AI system, requiring regular systematic review and updating. It shall comprise the following steps:\n\n(a) the identification and analysis of the known and the reasonably foreseeable risks that the high-risk AI system can pose to health, safety or fundamental rights when the high-risk AI system is used in accordance with its intended purpose;\n\n(b) the estimation and evaluation of the risks that may emerge when the high-risk AI system is used in accordance with its intended purpose, and under conditions of reasonably foreseeable misuse;\n\n(c) the evaluation of other risks possibly arising, based on the analysis of data gathered from the post-market monitoring system referred to in Article 72;\n\n(d) the adoption of appropriate and targeted risk management measures designed to address the risks identified pursuant to point (a)." }, { "requirementID": "9.3 Risk Management System", "requirementText": "The risks referred to in this Article shall concern only those which may be reasonably mitigated or eliminated through the development or design of the high-risk AI system, or the provision of adequate technical information." }, { "requirementID": "9.4 Risk Management System", "requirementText": "The risk management measures referred to in paragraph 2, point (d), shall give due consideration to the effects and possible interaction resulting from the combined application of the requirements set out in this Section, with a view to minimising risks more effectively while achieving an appropriate balance in implementing the measures to fulfil those requirements." }, { "requirementID": "9.5 Risk Management System", "requirementText": "The risk management measures referred to in paragraph 2, point (d), shall be such that the relevant residual risk associated with each hazard, as well as the overall residual risk of the high-risk AI systems is judged to be acceptable.\n\nIn identifying the most appropriate risk management measures, the following shall be ensured:\n\n(a) elimination or reduction of risks identified and evaluated pursuant to paragraph 2 in as far as technically feasible through adequate design and development of the high-risk AI system;\n\n(b) where appropriate, implementation of adequate mitigation and control measures addressing risks that cannot be eliminated;\n\n(c) provision of information required pursuant to Article 13 and, where appropriate, training to deployers.\n\nWith a view to eliminating or reducing risks related to the use of the high-risk AI system, due consideration shall be given to the technical knowledge, experience, education, the training to be expected by the deployer, and the presumable context in which the system is intended to be used." }, { "requirementID": "9.6 Risk Management System", "requirementText": " High-risk AI systems shall be tested for the purpose of identifying the most appropriate and targeted risk management measures. Testing shall ensure that high-risk AI systems perform consistently for their intended purpose and that they are in compliance with the requirements set out in this Section." }, { "requirementID": "9.9 Risk Management System", "requirementText": "When implementing the risk management system as provided for in paragraphs 1 to 7, providers shall give consideration to whether in view of its intended purpose the high-risk AI system is likely to have an adverse impact on persons under the age of 18 and, as appropriate, other vulnerable groups." }, { "requirementID": "9.10 Risk Management System", "requirementText": "For providers of high-risk AI systems that are subject to requirements regarding internal risk management processes under other relevant provisions of Union law, the aspects provided in paragraphs 1 to 9 may be part of, or combined with, the risk management procedures established pursuant to that law." }, { "requirementID": "55.8 Obligations of Providors of General-Purpose AI models with Systemic Risk", "requirementText": "In addition to the obligations listed in Articles 53 and 54, providers of general-purpose AI models with systemic risk shall:\n(a) perform model evaluation in accordance with standardised protocols and tools reflecting the state of the art, including conducting and documenting adversarial testing of the model with a view to identifying and mitigating systemic risks;\n(b) assess and mitigate possible systemic risks at Union level, including their sources, that may stem from the development, the placing on the market, or the use of general-purpose AI models with systemic risk;\n(c) keep track of, document, and report, without undue delay, to the AI Office and, as appropriate, to national competent authorities, relevant information about serious incidents and possible corrective measures to address them;\n(d) ensure an adequate level of cybersecurity protection for the general-purpose AI model with systemic risk and the physical infrastructure of the model." }, { "requirementID": "55.9 Obligations of Providors of General-Purpose AI models with Systemic Risk", "requirementText": "Providers of general-purpose AI models with systemic risk may rely on codes of practice within the meaning of Article 56 to demonstrate compliance with the obligations set out in paragraph 1 of this Article, until a harmonised standard is published. Compliance with European harmonised standards grants providers the presumption of conformity to the extent that those standards cover those obligations. Providers of general-purpose AI models with systemic risks who do not adhere to an approved code of practice or do not comply with a European harmonised standard shall demonstrate alternative adequate means of compliance for assessment by the Commission." } ] } }, "European Commission": { "Ethics guidelines for trustworthy AI": { "link": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirements": [ { "requirementID": "1.2.1 Resilience to attack and security", "requirementText": "Systems and data can also become corrupted by malicious intention or by exposure to unexpected situations. Insufficient security processes can also result in erroneous decisions or even physical harm. For AI systems to be considered secure, possible unintended applications of the AI system (e.g. dual-use applications) and potential abuse of the system by malicious actors should be taken into account, and steps should be taken to prevent and mitigate these. " }, { "requirementID": "1.7.2 Minimisation and Reporting of Negative Impacts", "requirementText": "Both the ability to report on actions or decisions that contribute to a certain system outcome, and to respond to the consequences of such an outcome, must be ensured. Identifying, assessing, documenting and minimising the potential negative impacts of AI systems is especially crucial for those (in)directly affected. Due protection must be available for whistle-blowers, NGOs, trade unions or other entities when reporting legitimate concerns about an AI system. The use of impact assessments (e.g. red teaming or forms of Algorithmic Impact Assessment) both prior to and during the development, deployment and use of AI systems can be helpful to minimise negative impact. These assessments must be proportionate to the risk that the AI systems pose." }, { "requirementID": "2.1.2 Ethics and Rule of Law by Design", "requirementText": "Methods to ensure values-by-design provide precise and explicit links between the abstract principles which the system is required to respect and the specific implementation decisions. The idea that compliance with norms can be implemented into the design of the AI system is key to this method. Companies are responsible for identifying the impact of their AI systems from the very start, as well as the norms their AI system ought to comply with to avert negative impacts. Different “by-design” concepts are already widely used, e.g. privacy-by-design and securityby-design. As indicated above, to earn trust AI needs to be secure in its processes, data and outcomes, and should be designed to be robust to adversarial data and attacks. It should implement a mechanism for fail-safe shutdown and enable resumed operation after a forced shut-down (such as an attack)." } ] } }, "Google": { "Secure AI Framework": { "link": "https://www.saif.google/secure-ai-framework", "requirements": [ { "requirementID": "Risk Governance", "requirementText": "Inventory, measure, and monitor residual risk to AI in your organization." } ] } }, "ICO": { "Guidance on the AI Auditing Framework - Draft guidance for consultation ": { "link": "https://ico.org.uk/media2/about-the-ico/consultations/2617219/guidance-on-the-ai-auditing-framework-draft-for-consultation.pdf", "requirements": [ { "requirementID": "How should we set a meaningful risk appetite?", "requirementText": "To manage the risks to individuals that arise from the processing of personal data in your AI systems, it is important that you develop a mature understanding and articulation of fundamental rights, risks, and how to balance these and other interests. Ultimately, it is necessary for you to:\n• assess the risks to individuals’ rights that your use of AI poses;\n• determine how you need to address these; and\n• establish the impact this has on your use of AI.\nYou should ensure your approach fits both your organisation and the circumstances of your processing. Where appropriate, you should also use risk assessment frameworks." }, { "requirementID": "What do we need to consider when undertaking data protection impact assessments for AI?", "requirementText": "DPIAs are a key part of data protection law’s focus on accountability and data protection by design. \nYou should not see DPIAs as a mere box ticking compliance exercise. They can effectively act as roadmaps for you to identify and control the risks to rights and freedoms that use of AI can pose. They are also a perfect opportunity for you to consider and demonstrate your accountability for the decisions you make in the design or procurement of AI systems." }, { "requirementID": "How do we identify and assess risks?", "requirementText": "The DPIA process will help you to objectively identify the relevant risks. You should assign a score or level to each risk, measured against the likelihood and the severity of the impact on individuals.\nThe use of personal data in the development and deployment of AI systems may not just pose risks to individuals’ information rights. When considering sources of risk, a DPIA should consider the potential impact of material and non-material damage or harm on individuals.\nFor instance, machine learning systems may reproduce discrimination from\nhistoric patterns in data, which could fall foul of equalities legislation. Similarly, AI systems that stop content being published based on the analysis of the creator’s personal data could impact their freedom of expression. In such contexts, you should consider the relevant legal frameworks beyond data protection. " }, { "requirementID": "How do we identify mitigating measures?", "requirementText": "Against each identified risk, you should consider options to reduce the level of assessed risk further. Examples of this could be data minimisation or providing opportunities for individuals to opt out of the processing.\n\nYou should ask your DPO for advice when considering ways to reduce or avoid risk, and you should record in your DPIA whether your chosen measure reduces or eliminates the risk in question. " }, { "requirementID": "How do we conclude our DPIA?", "requirementText": "You should record:\n• what additional measures you plan to take;\n• whether each risk has been eliminated, reduced or accepted;\n• the overall level of ‘residual risk’ after taking additional measures\n• the opinion of your DPO, if you have one; and\n• whether you need to consult the ICO." }, { "requirementID": "What’s different about security in AI compared to ‘traditional’ technologies?", "requirementText": "Some of the unique characteristics of AI mean compliance with data protection law’s security requirements can be more challenging than with other, more established technologies, both from a technological and human perspective.\nFrom a technological perspective, AI systems introduce new kinds of complexity not found in more traditional IT systems that you may be used to using. Depending on the circumstances, your use of AI systems is also likely to rely heavily on third party code and/or relationships with suppliers. Also, your existing systems need to be integrated with several other new and IT components, which are also intricately connected.\nThis complexity may make it more difficult to identify and manage some security risks, and may increase others, such as the risk of outages. " }, { "requirementID": "Preventative Controls - 13", "requirementText": "Document a DPIA, including thorough assessment of the security risks and the mitigants / controls to reduce the likelihood and impact of an attack." } ] } }, "IMDA": { "Model AI Governance Framework for Agentic AI": { "link": "https://www.imda.gov.sg/-/media/imda/files/about/emerging-tech-and-research/artificial-intelligence/mgf-for-agentic-ai.pdf", "requirements": [ { "requirementID": "2.1.1 Determine suitable use cases for agent deployment", "requirementText": "Risk identification and assessment is the first step when considering if an agentic use case is suitable for development or deployment. Risk is a function of likelihood (probability of the risk manifesting) and impact (severity of impact if the risk manifests)." } ] } }, "ISO": { "42001:2023 - Information technology — Artificial intelligence — Management system": { "link": "https://www.iso.org/standard/42001", "requirements": [ { "requirementID": "6.1", "requirementText": "Actions to address risks and opportunities" }, { "requirementID": "8.1", "requirementText": "Operational planning and control" }, { "requirementID": "8.2", "requirementText": "AI risk assessment" }, { "requirementID": "8.3", "requirementText": "AI risk treatment" }, { "requirementID": "8.4", "requirementText": "AI system impact assessment" } ] } }, "ISO/IEC": { "DIS 27090": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:27090:dis:ed-1:v1:en", "requirements": [ { "requirementID": "7.4", "requirementText": "Model and Mitigation Deterioration Over Time" } ] }, "TR 27091": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:27091:dis:ed-1:v1:en", "requirements": [ { "requirementID": "6.3", "requirementText": "Other privacy risks to AI systems" } ] }, "TR 27563:2023": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:tr:27563:ed-1:v1:en", "requirements": [ { "requirementID": "7.3", "requirementText": "Identify security and privacy concerns" }, { "requirementID": "7.4", "requirementText": "Identify security and privacy risks" }, { "requirementID": "7.5", "requirementText": "Identify security and privacy controls" }, { "requirementID": "7.6", "requirementText": "Identify security and privacy assurance concerns" }, { "requirementID": "7.7", "requirementText": "Identify security and privacy plan requirements" } ] }, "TS 42119-2:2025": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:ts:42119:-2:ed-1:v1:en", "requirements": [ { "requirementID": "6", "requirementText": "Identifying risks in AI systems" } ] } }, "METI (Japan)": { "Governance Guidelines for Implementation of AI Principles": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20220128_2.pdf", "requirements": [ { "requirementID": "Action Target 1-3", "requirementText": "Companies that develop and operate AI systems should, under the leadership of top management, evaluate and re-evaluate in a timely manner their AI proficiency based on the extent of the company’s experience in developing and operating AI systems, the number of employees, including engineers, involved in the development and operation of AI systems and their degree of experience, and the degree of AI literacy of these employees with respect to AI technology and ethics, except in situations where a company assesses negative impacts of their AI system are minor based on analyses of Action Targets 1-1 and 1-2 in light of the company’s business domain and scale, etc. If the negative impacts are assessed to be minor and no evaluation of AI proficiency is carried out, companies should be prepared to explain their rationale to their stakeholders." }, { "requirementID": "Action Target 3-1", "requirementText": "Companies that develop and operate AI systems should, under the leadership of top management, identify a gap between AI governance goals and current state in the AI systems that they are developing and operating, and if any negative impacts are found upon evaluating the impacts resulting from the gap, determine whether or not the negative impacts would be acceptable, taking into account their severity, scope, and frequency of occurrence. They should incorporate processes that prompt a reexamination of how the AI systems should be developed and operated in an appropriate stage such as during AI system design, development, before they are used, and after their usage begins, to address cases where the negative impacts are found not to be acceptable. Those in operations positions should make these processes concrete. In addition, those who are not directly involved in the development and operation of AI systems should be included in the gap analysis between AI governance goals and current state. It should be noted that it would not be appropriate to stop the development or provision of AI simply because a gap was found. As such, gap analysis is merely a step for evaluating negative impacts and simply serve as a starting point for improvement." }, { "requirementID": "Action Target 3-1-1", "requirementText": "Companies that develop and operate AI systems should, under the leadership of top management, check whether a standard gap analysis process in their industry is available and incorporate it into their own process if such a process is available." }, { "requirementID": "Action Target 3-4-2", "requirementText": "Companies that develop and operate AI systems should, under the leadership of top management, consider defining response guidelines and plans so that upon occurrence of an AI incident or dispute, they can promptly give an explanation to AI system users, identify the extent of the impact and damage, clarify legal responsibilities, consider relief measures and measures to prevent the spread of damage and recurrence, or take other relevant actions. Further, they should consider conducting rehearsal exercise relevant to such guidelines and plans, as appropriate." } ] } }, "MIC/METI (Japan)": { "AI Guidelines for Business": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirements": [ { "requirementID": "Human-Centric - 2 (b)", "requirementText": "When developing, providing, or using an AI system or service, pay attention and take necessary countermeasures against the risk of heavy dependence on AI, such as automated biases." }, { "requirementID": "Safety - 1 (d)", "requirementText": "Conduct appropriate risk analyses to take countermeasures against risks (avoidance, mitigation, transference, or acceptance)." }, { "requirementID": "Safety - 1 (e)", "requirementText": "If there are potential hazards to the lives, bodies, properties, and minds of humans and the environment, organize measures to be taken in advance and offer related information to stakeholders. Clearly specify measures that should be taken by relevant stakeholders and the terms of use." }, { "requirementID": "Ensuring security - 1 (c)", "requirementText": "Bearing in mind that relevant stakeholders might make unexpected judgments by mixing detailed information into inference target data, recognize that vulnerabilities cannot be completely eliminated from AI systems and services." }, { "requirementID": "Ensuring security - 2", "requirementText": "New methods for attacking AI systems and services from the outside are increasing on a daily basis. In order to address those risks, check the matters to be noted." }, { "requirementID": "Accountability - 2", "requirementText": "Provide and explain information on how AI business actors conform to common guiding principles regularly to stakeholders, including suppliers, according to their knowledge and competence. This information summarizes, for example, the following items:\nGeneral:\n- Whether any risk is found that prevents the common guiding principles from being implemented, to what extent it prevents the implementation of those guiding principles\n- Implementation progress of the common guiding principles\nHuman-centric:\n- How disinformation is considered, and how diversity, inclusion, user support, and sustainability are ensured\nSafety:\n- Known risks relating to AI systems and services, countermeasures against them, and how to ensure safety against them\nFairness:\n- Possibility that technological elements forming AI models will introduce bias. Those elements may include training data, AI model training process, prompts expected to be entered by AI business users or non-business users, and reference information and collaborating external services used by AI models for inference.\nPrivacy protection:\n- Risks of infringements of privacy of AI business actors or stakeholders entailed by the AI system or service, countermeasures against those risks, and actions expected to be taken when the privacy breach actually occurred.\nEnsuring security:\n- Conformity to standards required to facilitate collaboration between AI systems and services or with other systems if such collaboration occurs\n- Any risks that may occur when the AI system or service collaborates with other AI systems and services via the Internet, and measures to be taken against the risks" }, { "requirementID": "Accountability - 5 (a)", "requirementText": "As necessary, establish and publicly report policies, including those created by each AI business actor on AI governance or privacy in relation to risk management or safety assurance associated with the use of AI systems and services. Those policies involve social responsibilities, including sharing visions with and giving out and providing information to society and general citizens." } ] } }, "Microsoft": { "Cloud Adoption Framework - Secure AI": { "link": "https://learn.microsoft.com/en-us/azure/cloud-adoption-framework/scenarios/ai/secure", "requirements": [ { "requirementID": "Discover AI security risks\n4 - Conduct periodic risk assessments", "requirementText": "New threats emerge as AI models, usage patterns, and threat actors evolve over time. Regular assessments ensure your security posture adapts to changing risk landscapes. Run recurring assessments to identify vulnerabilities in models, data pipelines, and deployment environments, and use assessment findings to guide your risk mitigation priorities." } ] }, "Responsible AI Standard": { "link": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirements": [ { "requirementID": "A2.1", "requirementText": "Review defined Restricted Uses to determine whether the system meets the definition of any Restricted Use. If it does, document this in the Impact Assessment, and follow the requirements for the Restricted Use." }, { "requirementID": "A2.2", "requirementText": "Answer prompts in the Impact Assessment template to determine whether the system meets the definition of a Sensitive Use. If it does, report it to the Office of Responsible AI, and follow any additional requirements resulting from a Sensitive Uses review." }, { "requirementID": "A2.3", "requirementText": "Review your systems at least annually against the definitions for Sensitive Uses and Restricted Uses. If there are systems that meet the criteria for Sensitive Uses, report them to the Office of Responsible AI. If there are systems that meet the criteria for Restricted Uses, notify the Office of Responsible AI." }, { "requirementID": "A5.7", "requirementText": "If there are Responsible Release Criteria for metrics or rubrics that have not been met, consult with the reviewers named in the Impact Assessment, and in the case of Sensitive Uses, with the Office of Responsible AI, to develop a plan detailing how the gap will be managed until it can be closed. Document that plan." }, { "requirementID": "T3.1", "requirementText": "Identify stakeholders who will use or be exposed to the system, in accordance with the Impact Assessment requirements. Document these stakeholders using the Impact Assessment template." }, { "requirementID": "F3.7", "requirementText": "Publish information for customers about these risks involving identified demographic groups. When the system is a platform service made available to external customers or partners, include this information in the required Transparency Note." }, { "requirementID": "RS2.1", "requirementText": "Define predictable failures, including false positive and false negative results for the system as a whole and how they would impact stakeholders for each intended use. Use the Impact Assessment template to document any adverse impacts of these failures on stakeholders." }, { "requirementID": "RS2.2", "requirementText": "For each case of a predictable failure likely to have an adverse impact on a stakeholder, document the failure management approach:\n1) When possible, design and build the system to avoid this failure. Describe the design solution. Estimate the time range for resolving predictable failures for each designed solution or indicate that the failure will be prevented by design.\n2) When a failure cannot be prevented by design, build a fallback option that may be used when this failure occurs. Describe the fallback option and document the estimated time required to invoke and use the fallback option.\n3) Provide training and documentation for stakeholders accountable for system oversight that supports their resolution of the failure. Describe the documentation and training." }, { "requirementID": "RS3.3", "requirementText": "When new uses, critical operational factors, or changes in the supported range of an operational factor are identified, determine whether any new use or operational factor can be supported with the existing system, will be supported but require additional work, or will not be supported.\n• When new uses or operational factors identified are to be supported, evaluate the updated system in accordance with requirement RS1.6, add the new intended use to the Impact Assessment, and publish updated communication in accordance with requirement RS1.9.\n• When these new uses or operational factor range changes cannot or will not be accommodated to ensure reliable and safe performance of the system update customer documentation described in RS1.9 to include the new use as an unsupported use.\nWhen the system is a platform service made available to external customers or partners, include this information in the required Transparency Note." }, { "requirementID": "RS3.4", "requirementText": "When a system is to be used for a Sensitive Use that imposes qualification or quality control requirements beyond the intended uses and/or operational factor ranges, conduct an evaluation specific to this use. If the required Responsible Release Criteria cannot be met, the Office of Responsible AI will review the results and decide how to proceed. Document any changes to the Responsible Release Criteria and document the results of evaluation." }, { "requirementID": "RS3.6", "requirementText": "If there are targets in Ongoing Evaluation Checkpoints that are no longer satisfied, consult with named reviewers, and in the case of Sensitive Uses, with the Office of Responsible AI, to develop and implement a plan to close any gaps. Document the process, its results, and conclusions." } ] } }, "MITRE": { "SAFE-AI": { "link": "https://atlas.mitre.org/pdf-files/SAFEAI_Full_Report.pdf", "requirements": [ { "requirementID": "Vulnerability exploit", "requirementText": "Code development and testing practices for AI-enabled systems do not always conform to traditional software development practices. Consequently, assessing AI system vulnerabilities may raise unexpected challenges (e.g., in some cases it may be difficult to even identify what to test). These challenges are of course a prime opportunity for attackers to exploit gaps in the vulnerability assessment and initiate attacks. Avoiding these undesirable outcomes requires stringent approaches to vulnerability assessment and monitoring. All known potential threats, vulnerabilities, and attack vectors associated with an AI-enabled system must be identified early during the design phase (e.g., by using ATLAS) and the risks must be managed. It is critical to define metrics and procedures for detecting, tracking, and measuring known risks, errors, incidents, or negative impacts. Metrics should also account for known AI design and implementation failure modes associated with properties like brittleness. The deployed AI-enabled system must be continuously tested for errors or vulnerabilities" } ] } }, "Multi Agency": { "Guidelines for secure AI system development": { "link": "https://www.ncsc.gov.uk/files/Guidelines-for-secure-AI-system-development.pdf", "requirements": [ { "requirementID": "Model the threats to your system", "requirementText": "As part of your risk management process, you apply a holistic process to assess the threats to your system, which includes understanding the potential impacts to the system, users, organisations, and wider society if an AI component is compromised or behaves unexpectedly. This process involves assessing the impact of AI-specific threats and documenting your decision making.\n\nYou recognise that the sensitivity and types of data used in your system may influence its value as a target to an attacker. Your assessment should consider that some threats may grow as AI systems increasingly become viewed as high value targets, and as AI itself enables new, automated attack vectors." } ] } }, "NCSC/NSA/CISA etc": { "AI Data Security\n": { "link": "https://media.defense.gov/2025/May/22/2003720601/-1/-1/0/CSI_AI_DATA_SECURITY.PDF", "requirements": [ { "requirementID": "1.10 Conduct ongoing data security risk assessments", "requirementText": "Conduct ongoing risk assessments using industry-standard frameworks, such as the NIST SP 800-3r2, Risk Management Framework (RMF) [4] [21], and the NIST AI 100-1, Artificial Intelligence RMF [3]. These assessments should evaluate the AI data security landscape, identify risks, and prioritize actions to minimize security incidents. Continuously improve data security measures to keep pace with evolving threats and vulnerabilities, learn from security incidents, stay up to date with emerging technologies, and maintain a robust security posture." } ] } }, "NIST": { "AI 800-1": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirements": [ { "requirementID": "Practice 1.1: Anticipate model capabilities - 5", "requirementText": "Assess the degree of uncertainty in these estimates based on the difference between the proxy model and the planned model, how those differences are expected to affect their capabilities, and the reliability and completeness of the evaluations available for proxy models." }, { "requirementID": "Practice 1.1: Anticipate model capabilities - 7", "requirementText": "If capability forecasts are uncertain and include the possibility that the model may require increased risk mitigations, consider increasing the frequency of capability measurements during the development process (Objective 4) and expanding the organization’s planned risk mitigation measures (Objective 5)." }, { "requirementID": "Practice 1.2 Create threat profiles - 3", "requirementText": "Use real data, case studies, or expert opinions to inform threat profiles and help identify gaps." }, { "requirementID": "Practice 1.2 Create threat profiles - 4", "requirementText": "Develop a plan for identifying and adding threat profiles to this list as future research reveals new potential or ongoing misuse risks (Objective 2)." }, { "requirementID": "Practice 1.3: Conduct risk assessments - 1", "requirementText": "Consider both quantitative and qualitative assessments of likelihood and impact, if possible. " }, { "requirementID": "Practice 1.3: Conduct risk assessments - 2", "requirementText": "Account for alternative tools already available to threat actors, such as existing models or other digital tools, and assess the model’s marginal risk of misuse relative to that baseline." }, { "requirementID": "Practice 1.3: Conduct risk assessments - 3", "requirementText": "Consider the new risk the model may introduce, such as the model’s ability to help an actor increase the scale, prevalence, or frequency, decrease the cost, or improve the effectiveness or efficiency of their malicious activity." }, { "requirementID": "Practice 1.3: Conduct risk assessments - 4", "requirementText": "Consider malicious actors’ potential motivations, willingness, and capacity to enact harm, as well as the number of malicious actors that may exist." }, { "requirementID": "Practice 1.3: Conduct risk assessments - 5", "requirementText": "Account for existing mitigations and barriers, such as limitations on access to physical resources needed to enact harm, and the level of societal preparedness to defend against that harm." }, { "requirementID": "Practice 1.3: Conduct risk assessments - 6", "requirementText": "Use information about the real-world impact and use of proxy models to inform the assessment; update these assessments as new information about real-world impact materializes." }, { "requirementID": "Practice 1.3: Conduct risk assessments - 7", "requirementText": "Update estimates if changes in the broader ecosystem either increase or decrease vulnerability to potential harms." }, { "requirementID": "Practice 1.3: Conduct risk assessments - 8", "requirementText": "Recognize areas of uncertainty and sensitivity and account for these areas when communicating and using impact assessments." }, { "requirementID": "Practice 2.1. Map anticipated model capabilities to appropriate risk mitigations to manage misuse risk - 1", "requirementText": "Consider the costs and benefits of planned risk mitigations, using both qualitative and quantitative comparisons." }, { "requirementID": "Practice 2.1. Map anticipated model capabilities to appropriate risk mitigations to manage misuse risk - 2", "requirementText": "Consider the risk in the context of organizational risk tolerances, which may reflect legal and regulatory obligations." }, { "requirementID": "Practice 2.1. Map anticipated model capabilities to appropriate risk mitigations to manage misuse risk - 3", "requirementText": "Identify and articulate evidence to justify the adequacy of planned risk mitigations." }, { "requirementID": "Practice 2.1. Map anticipated model capabilities to appropriate risk mitigations to manage misuse risk - 4", "requirementText": "Refine planned methods to mitigate misuse risk periodically based on adjustments to identified threat profiles, changes in factors that affect risk tolerance (such as increases in expected benefits), and information about real-world performance." }, { "requirementID": "Practice 2.1. Map anticipated model capabilities to appropriate risk mitigations to manage misuse risk - 5", "requirementText": "As measurement and real-world monitoring is performed, continue to re-assess the safeguards necessary to manage misuse risk for each particular capability." }, { "requirementID": "Practice 2.2 Establish an organizational plan to manage misuse risk - 4", "requirementText": "Plan to monitor evidence of real-world misuse and potential risk. Develop processes for adjusting these organizational plans, as well as deployment or development approaches, as necessary." }, { "requirementID": "Practice 3.1: Assess misuse risk from threat actors gaining unauthorized access to the model - 1", "requirementText": "Consider possible threat actors’ motivations and level of sophistication related to gaining unauthorized access to the model." }, { "requirementID": "Practice 3.1: Assess misuse risk from threat actors gaining unauthorized access to the model - 3", "requirementText": "Consider the threat posed by insiders, such as an individual involved in developing or deploying the model who may behave maliciously or collaborate with an external attacker." }, { "requirementID": "Practice 3.2: Maintain security practices sufficient to prevent unauthorized access - 4", "requirementText": "Re-assess the risk of unauthorized access as security practices are implemented." }, { "requirementID": "Practice 5.2: Assess misuse risk based on implemented safeguards - 2", "requirementText": "Estimate misuse risk based on appropriate measurements conducted under Objective 4 and comparisons to proxy models conducted in Practice 1.1." }, { "requirementID": "Practice 5.3: Adopt appropriate deployment strategies based on misuse risk assessments - 1", "requirementText": "Assess residual risk by incorporating the overall assessed misuse risk from the selected deployment strategy and mitigations from implemented safeguards." }, { "requirementID": "Practice 5.3: Adopt appropriate deployment strategies based on misuse risk assessments - 3", "requirementText": "Consider whether additional safeguards are feasible to implement prior to deployment, whether additional time could be used to carry out a more reliable estimate of risk, or whether a more limited deployment may be more appropriate given the level of assessed risk." }, { "requirementID": "Practice 5.3: Adopt appropriate deployment strategies based on misuse risk assessments - 4", "requirementText": "Consider leaving a buffer between the estimated level of risk—given the implemented safeguards and the deployment strategy—and the associated anticipated real-world risk (hereafter referred to as a ‘margin of safety’). This margin of safety could incorporate how threat actors may continue to acquire new knowledge about how to misuse or augment the model after it is deployed26 and how to circumvent safeguards. Consider a larger margin of safety to manage risks that are more severe or less certain." } ] }, "AI RMF 1.0": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirements": [ { "requirementID": "GOVERN 1.3", "requirementText": "Processes, procedures, and practices are in place to determine the needed level of risk management activities based on the organization’s risk tolerance." }, { "requirementID": "GOVERN 1.4", "requirementText": "The risk management process and its outcomes are established through transparent policies, procedures, and other controls based on organizational risk priorities." }, { "requirementID": "GOVERN 1.5", "requirementText": "Ongoing monitoring and periodic review of the risk management process and its outcomes are planned and organizational roles and responsibilities clearly defined, including determining the frequency of periodic review." }, { "requirementID": "GOVERN 2.1", "requirementText": "Roles and responsibilities and lines of communication related to mapping, measuring, and managing AI risks are documented and are clear to individuals and teams throughout the organization." }, { "requirementID": "GOVERN 3.1", "requirementText": "Decision-making related to mapping, measuring, and managing AI risks throughout the lifecycle is informed by a diverse team (e.g., diversity of demographics, disciplines, experience, expertise, and backgrounds)." }, { "requirementID": "GOVERN 4.2", "requirementText": "Organizational teams document the risks and potential impacts of the AI technology they design, develop, deploy, evaluate, and use, and they communicate about the impacts more broadly." }, { "requirementID": "MAP 1.5", "requirementText": "Organizational risk tolerances are determined and documented." }, { "requirementID": "MAP 3.2", "requirementText": "Potential costs, including non-monetary costs, which result from expected or realized AI errors or system functionality and trustworthiness – as connected to organizational risk tolerance – are examined and documented." }, { "requirementID": "MAP 4.2", "requirementText": "Internal risk controls for components of the AI system, including third-party AI technologies, are identified and documented." }, { "requirementID": "MAP 5.1", "requirementText": "Likelihood and magnitude of each identified impact (both potentially beneficial and harmful) based on expected use, past uses of AI systems in similar contexts, public incident reports, feedback from those external to the team that developed or deployed the AI system, or other data are identified and documented." }, { "requirementID": "MEASURE 1.1", "requirementText": "Approaches and metrics for measurement of AI risks enumerated during the MAP function are selected for implementation starting with the most significant AI risks. The risks or trustworthiness characteristics that will not – or cannot – be measured are properly documented." }, { "requirementID": "MEASURE 2.8", "requirementText": "Risks associated with transparency and account- ability – as identified in the MAP function – are examined and documented." }, { "requirementID": "MEASURE 2.10", "requirementText": "Privacy risk of the AI system – as identified in the MAP function – is examined and documented." }, { "requirementID": "MEASURE 3.1", "requirementText": "Approaches, personnel, and documentation are in place to regularly identify and track existing, unanticipated, and emergent AI risks based on factors such as intended and actual performance in deployed contexts." }, { "requirementID": "MEASURE 3.2", "requirementText": "Risk tracking approaches are considered for settings where AI risks are difficult to assess using currently available measurement techniques or where metrics are not yet available." }, { "requirementID": "MANAGE 1.2", "requirementText": "Treatment of documented AI risks is prioritized based on impact, likelihood, and available resources or methods." }, { "requirementID": "MANAGE 1.3", "requirementText": "Responses to the AI risks deemed high priority, as identified by the MAP function, are developed, planned, and documented. Risk response options can include mitigating, transferring, avoiding, or accepting." }, { "requirementID": "MANAGE 1.4", "requirementText": "Negative residual risks (defined as the sum of all unmitigated risks) to both downstream acquirers of AI systems and end users are documented." }, { "requirementID": "MANAGE 2.3", "requirementText": "Procedures are followed to respond to and recover from a previously unknown risk when it is identified." } ] }, "IR 8596: Cybersecurity Framework Profile for Artificial Intelligence (Cyber AI Profile): NIST Community Profile": { "link": "https://csrc.nist.gov/pubs/ir/8596/iprd", "requirements": [ { "requirementID": "GV.RM-01", "requirementText": "Risk management objectives are established and agreed to by organizational stakeholders" }, { "requirementID": "GV.RM-02", "requirementText": "Risk appetite and risk tolerance statements are established, communicated, and maintained" }, { "requirementID": "GV.RM-03", "requirementText": "Cybersecurity risk management activities and outcomes are included in enterprise risk management processes" }, { "requirementID": "GV.RM-04", "requirementText": "Strategic direction that describes appropriate risk response options is established and communicated" }, { "requirementID": "GV.RM-05", "requirementText": "Lines of communication across the organization are established for cybersecurity risks, including risks from suppliers and other third parties" }, { "requirementID": "GV.RM-06", "requirementText": "A standardized method for calculating, documenting, categorizing, and prioritizing cybersecurity risks is established and communicated" }, { "requirementID": "GV.RM-07", "requirementText": "Strategic opportunities (i.e., positive risks) are characterized and are included in organizational cybersecurity risk discussions" }, { "requirementID": "GV.PO-01", "requirementText": "Policy for managing cybersecurity risks is established based on organizational context, cybersecurity strategy, and priorities and is communicated and enforced" }, { "requirementID": "GV.PO-02", "requirementText": "Policy for managing cybersecurity risks is reviewed, updated, communicated, and enforced to reflect changes in requirements, threats, technology, and organizational mission" }, { "requirementID": "GV.OV-01", "requirementText": "Cybersecurity risk management strategy outcomes are reviewed to inform and adjust strategy and direction" }, { "requirementID": "GV.OV-02", "requirementText": "The cybersecurity risk management strategy is reviewed and adjusted to ensure coverage of organizational requirements and risks" }, { "requirementID": "GV.OV-03", "requirementText": "Organizational cybersecurity risk management performance is evaluated and reviewed for adjustments needed" }, { "requirementID": "GV.SC-01", "requirementText": "A cybersecurity supply chain risk management program, strategy, objectives, policies, and processes are established and agreed to by organizational stakeholders" }, { "requirementID": "ID.RA-02", "requirementText": "Cyber threat intelligence is received from information sharing forums and sources" }, { "requirementID": "ID.RA-03", "requirementText": "Internal and external threats to the organization are identified and recorded" }, { "requirementID": "ID.RA-04", "requirementText": "Potential impacts and likelihoods of threats exploiting vulnerabilities are identified and recorded" }, { "requirementID": "ID.RA-05", "requirementText": "Threats, vulnerabilities, likelihoods, and impacts are used to understand inherent risk and inform risk response prioritization" }, { "requirementID": "ID.RA-06", "requirementText": "Risk responses are chosen, prioritized, planned, tracked, and communicated" }, { "requirementID": "ID.RA-07", "requirementText": "Changes and exceptions are managed, assessed for risk impact, recorded, and tracked" }, { "requirementID": "RC.RP-04", "requirementText": "Critical mission functions and cybersecurity risk management are considered to establish post-incident operational norms" } ] }, "SP 800-218A": { "link": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirements": [ { "requirementID": "PW.1.1", "requirementText": "Use forms of risk modeling – such as threat modeling, attack modeling, or attack surface mapping – to help assess the security risk for the software.\n\nIncorporate relevant AI model-specific vulnerability and threat types in risk modeling. Examples of these vulnerability and threat types include poisoning of training data, malicious code or other unwanted content in inputs and outputs, denial-of-service conditions arising from adversarial prompts, supply chain attacks, unauthorized information disclosure, theft of AI model weights, and misconfiguration of data pipelines. " }, { "requirementID": "RV.2.2", "requirementText": "Plan and implement risk responses for vulnerabilities.\n\nRisk responses for AI models should consider the time and expenses that may be associated with rebuilding them.\n\nEstablish and implement criteria and processes for when to stop using an AI model and when to roll back to a previous version and its components." } ] } }, "OECD": { "Due Diligence Guidance for Responsible AI": { "link": "https://www.oecd.org/content/dam/oecd/en/publications/reports/2026/02/oecd-due-diligence-guidance-for-responsible-ai_7831bb49/41671712-en.pdf", "requirements": [ { "requirementID": "Step 2.1 – Initial scoping of risks", "requirementText": "Carry out a scoping exercise to identify where risks may be present and where they may be most significant. \n\nMultiple frameworks exist at the international, regional and national level that describe risks related to the development and use of AI systems and recommend actions companies should take to address those risks. One objective of this guidance is to support enterprises’ implementation of other risk management frameworks. While the list of frameworks can inform a range of potentially relevant risks for an enterprise’s due diligence efforts, it is non-exhaustive and many of the risks overlap and may be linked to each other. Likewise, not all frameworks are relevant for every enterprise. Each enterprise is expected to identify its priority risk areas based on its individual circumstances.\n\nWhen prioritising the order in which risks are to be addressed (see Step 2.4), enterprises should take into account that certain risks are closely linked to or may enable others.\n\nAs with many new technologies, public and private malign actors may find ways to exploit AI systems. The significant dual-use potential of AI systems and ability to repurpose AI systems can lead to harmful uses even when their design was intended to be innocuous. " }, { "requirementID": "Step 2.2 – In-depth assessment of most significant risks", "requirementText": "Starting with the most significant areas of risk identified, carry out iterative and increasingly in-depth assessments of prioritised risks related to (1) the enterprise’s own activities and (2) the enterprise’s business relationships (e.g., suppliers, customers and users)." }, { "requirementID": "Step 2.3 – Assess involvement with the actual or potential impact (cause, contribute, directly linked)", "requirementText": "Assess the enterprise’s involvement with the actual or potential adverse impacts identified. Specifically assess whether the organisation or relevant business relationship caused (or would cause) the adverse impact; or contributed (or would contribute) to the adverse impact; or whether the adverse impact is (or would be) directly linked to its operations, products or services by a business relationship. An enterprise’s relationship to adverse impact is not static. It may change, for example as situations evolve and depending upon the degree to which due diligence and steps taken to address identified risks and adverse impacts decrease the risk of the impacts occurring." }, { "requirementID": "Step 2.4 – Prioritise the most significant (i.e., most salient) risks", "requirementText": "Drawing from the information obtained on actual and potential adverse impacts, prioritise the most significant (i.e., most salient) risks and adverse impacts for action, based on severity and likelihood. Prioritisation will be relevant where it is not possible to address all potential and actual adverse impacts immediately. Once the most significant adverse impacts are identified and dealt with, the enterprise should move on to address less significant foreseeable impacts.\n\nWhere the risk of adverse impacts is most significant will be specific to the enterprise. Thus, it is important for enterprises to demonstrate a credible prioritisation process.\n\nEngaging with relevant stakeholders, including workers, workers’ representatives and trade unions, on how to prioritise and publicly communicating the rationale behind how prioritisation decisions are made can be useful for establishing trust in the enterprise’s due diligence approach. In some cases, prioritisation may also be informed by domestic legal obligations. " }, { "requirementID": "Step 3.1 – Addressing risks that the enterprise causes or contributes to", "requirementText": "Cease activities that are causing or contributing to adverse impacts based on the enterprise’s assessment of its involvement with the impact. \n\nDevelop and implement plans to prevent and mitigate potential (future) adverse impacts." } ] } }, "Personal Data Protection Commission Singapore (PDPC)": { "Model Artificial Intelligence Governance Framework Second Edition": { "link": "https://www.pdpc.gov.sg/-/media/files/pdpc/pdf-files/resource-for-organisation/ai/sgmodelaigovframework2.pdf", "requirements": [ { "requirementID": "2. Risk management and internal controls - a)", "requirementText": "Organisations can consider implementing a sound system of risk management and internal controls that specifically addresses the risks involved in the deployment of the selected AI model." }, { "requirementID": "2. Risk management and internal controls - b) (i)", "requirementText": "Such measures include:\nUsing reasonable efforts to ensure that the datasets used for AI model training are adequate for the intended purpose, and to assess and manage the risks of inaccuracy or bias, as well as reviewing exceptions identified during model training. Virtually, no dataset is completely unbiased. Organisations should strive to understand the ways in which datasets may be biased and address this in their safety measures and deployment strategies." }, { "requirementID": "2. Risk management and internal controls - b) (iv)", "requirementText": "Reviewing the internal governance structure and measures when there are significant changes to organisational structure or key personnel involved." }, { "requirementID": "2. Risk management and internal controls - b) (v)", "requirementText": "Periodically reviewing the internal governance structure and measures to ensure their continued relevance and effectiveness." } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirements": [ { "requirementID": "6.1", "requirementText": "An Entity must create a defined Al strategy based on the Entity's needs and risk appetite. It must also be consistent with the Entity's relevant strategies and internal policies and processes." }, { "requirementID": "6.2", "requirementText": "An Entity must conduct a periodic review of its Al strategy at a time consistent with other strategic reviews." }, { "requirementID": "7.2", "requirementText": "The key responsibilities of the BOD include but are not limited to:\n- Approving the level of Al exposures to be tolerated in the overall risk framework.\n- Deciding whether an Entity's existing governance structures are fit for purpose.\n- Assigning clear lines of accountability and responsibility.\n- Ensuring adequate human resourcing of all Al functions." }, { "requirementID": "7.6", "requirementText": "An Entity must manage risks associated with the use of Al within the enterprise risk management structure." }, { "requirementID": "8.2.1", "requirementText": "Management of Al-related risks through regular audits covering regulatory compliance, governance, customer interactions, risk management process, systems, and control evaluation, and applying required mitigation controls." }, { "requirementID": "9.1", "requirementText": "An Entity must evaluate risks associated with deployment of Al within the organization." }, { "requirementID": "9.3", "requirementText": "An Entity must determine Al risk levels by conducting risk and criticality assessments of the process and functions that the Al System implementation is a part of or connected to." }, { "requirementID": "9.4", "requirementText": "An Entity must manage the process so that the Al risk assessment stays in line with overall process and function risk assessment." }, { "requirementID": "9.5", "requirementText": "An Entity must inform its Al risk score by using additional factors." }, { "requirementID": "9.5.1", "requirementText": "An Al System that allows no direct Human Oversight will probably be judged a higher-than-normal risk, while an Al System that works at delivering \"Al-assisted\" outcomes, where the final determination is always made by a human with relevant expertise, will likely be judged a lower risk category." }, { "requirementID": "9.5.2", "requirementText": "An Entity assessing the use of an Al System developed by a third-party vendor must consider the access to information allowed and the reputation of the vendor as important risk assessment variables." }, { "requirementID": "9.6", "requirementText": "The Entity will determine whether a specific Al System is \"High Risk\" using the definition in Section (2), and the risk evaluation and rating per clauses (9.1) to (9.5)." }, { "requirementID": "9.7", "requirementText": "Notwithstanding the determination made in clause (9.6), an Entity must classify an Al System as high-risk if or when there is a level of potential harm to natural persons from the system, namely with respect to:\n- Interactions determining consumer access to financial services offerings.\n- Internal organizational decisions that affect employees in a material manner.\n- Processing sensitive personal information." }, { "requirementID": "9.8", "requirementText": "An Entity, if it is the Provider, must have or develop the risk management system that specifically is designed to handle Al related risk with capacity to handle the profile of each Al System." }, { "requirementID": "10.4", "requirementText": "High-risk systems must be highlighted." }, { "requirementID": "12.3", "requirementText": "An Entity should conduct a risk assessment of the outsourcing service provider, including the location of the data when it is processed, stored, and transmitted, and any relevant vendor contracted by the third party." }, { "requirementID": "20.3", "requirementText": "An Entity should ensure that Customers are informed of products and/or services that utilize Al, the associated risks, and limitations of the technology:\n- Prior to providing the service initially (for non-high-risk systems like chatbots).\n- Each update of an Al System will be treated as a new version requiring Customer notification.\n- Use of Al can be disclosed in the general product description or terms of use in line with clause (17.2).\n- Each time Customers (or employees) interact with the service for High-Risk Al like credit, insurance, or employee issues." }, { "requirementID": "20.9", "requirementText": "An Entity should collect Customer consent to acceptance of the risks associated with the use of Al prior to providing the service. For non-high-risk Al Systems this could be a one-time event as above or in general terms of use." }, { "requirementID": "23.3", "requirementText": "An Entity must support its exemption request with a clear and documented business case or rationale." }, { "requirementID": "23.5", "requirementText": "An Entity must duly record in the Entity's exemptions Register any approved exemptions and assign an expiration date - the date by which the exemption will be mitigated or resolved by the Entity seeking the exemption." } ] } }, "SANS": { "Critical AI Security Guidelines": { "link": "https://sansorg.egnyte.com/dl/bvkYQxrW8QMj", "requirements": [ { "requirementID": "4.5 Modality", "requirementText": "Although powerful, multimodal implementations can increase the attack surface. Common sense and research suggest safety and alignment can prove inconsistent across different modalities. As an example, a text-only prompt that might have been considered unsafe could be allowed if the text were instead submitted as an image." }, { "requirementID": "4.6 Languages and Character Sets", "requirementText": "It has been shown that multilingual and multicharacter models can introduce new vulnerabilities and expand the attack surface. Multilingual jailbreak challenges have been observed when utilizing prompts in a language other than that used in the primary training data.12 Research has shown this can result in jailbreaking or providing instructions to deliberately attack vulnerable LLMs. The same is true for character sets, which have been shown to increase hallucinations and comprehension errors.13 Additional research highlights that when instructions involve Unicode characters outside the standard Latin or variants of other languages, a reduction in guardrail efficiency is observed." } ] } }, "SDAIA (Saudi Arabia)": { "AI Ethics Principles": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Documents/ai-principles.pdf", "requirements": [ { "requirementID": "Principle 5 – Reliability & Safety - Plan and Design - 2", "requirementText": "Planning to set out a robust and reliable AI system that works with different sets of inputs and situations is essential to prevent unintended harm and mitigate risks of system failures when positioned against unknown and unforeseen events." }, { "requirementID": "Principle 7 – Accountability & Responsibility - Plan and Design - 2", "requirementText": "Organizations can put in place additional instruments such as impact assessments, risk mitigation frameworks, audit and due diligence mechanisms, redress, and disaster recovery plans." } ] }, "Generative AI Guidelines": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Files/GenerativeAIPublicEN.pdf", "requirements": [ { "requirementID": "4.5 Privacy & Security - 3", "requirementText": "Assess the risks resulting from the use of the GenAI tool according to the AI ethics principles; little or no risk, limited risk, high risk, unacceptable risk." }, { "requirementID": "4.5 Privacy & Security - 5", "requirementText": "The privacy impact assessment and risk management assessment should be continuously revisited to ensure that societal and ethical considerations are regularly evaluated." } ] } }, "Smart Dubai (UAE)": { "AI Ethics Principles & Guidelines": { "link": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirements": [ { "requirementID": "1.2.2.2", "requirementText": "AI operator organisations should identify the likely impact of incorrect automated decisions on AI subjects and, in the case where incorrect decisions are likely to cause significant cost or inconvenience, consider mitigating measures." }, { "requirementID": "1.2.2.3", "requirementText": "AI operator organisations should consider internal risk assessments or ethics frameworks as a means to facilitate the identification of risks and mitigating measures" } ] } }, "U.S. Department of Health & Human Services": { "Trustworthy AI (TAI) Playbook: Executive Summary": { "link": "https://www.hhs.gov/sites/default/files/hhs-trustworthy-ai-playbook-executive-summary.pdf", "requirements": [ { "requirementID": "Safe / Secure", "requirementText": "AI systems should be protected from risks (including Cyber) that may directly or indirectly cause physical and/or digital harm to any individual, group, or entity" } ] } } }, "Principle 4": { "Central Bank of the UAE": { "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E": { "link": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirements": [ { "requirementID": "2. Governance and Accountability - b", "requirementText": "Senior management and the Board of Directors of LFIs should be responsible and accountable for AI and ML systems and outcomes, model selection/development, deployment, accountability, appropriate human resourcing and oversight and monitoring and management on an on-going basis." }, { "requirementID": "6. Continuous Monitoring and Review - d", "requirementText": "LFIs should remain responsible for outsourced AI functions and should consider: appropriate contractual rights with respect to audit and information rights from providers, be made aware of any material developments with the AI provider, appropriate termination/cease provisions, data protection, cyber security, performance guarantees, its compliance with laws/regulations and standards and any material developments with regard to the AI being outsourced/utilized. " }, { "requirementID": "6. Continuous Monitoring and Review - e", "requirementText": "LFIs should at all times retain the clear and immediate ability, with human intervention, to cease use of an AI model system, technology or application deployed or utilized." }, { "requirementID": "7. Human Oversight and Consumer Protection - a", "requirementText": "LFIs should ensure that AI and ML systems operate under meaningful human oversight and judgement, particularly for decisions that have significant implications for consumers and in respect of the ongoing selection of, determination as to third party providers of, deployment of and ongoing monitoring and general use of AI. Human oversight may be exercised through different models:\n(i) Human-in-the-loop – where a AI provides recommendations but a human decision maker retains full authority to approve or reject the outcome; \n(ii) Human-on-the-loop – where the AI works autonomously for routine tasks, while a human monitors outcomes and can intervene where necessary; \n(iii) Human-out-of-the-loop – where the AI operates without direct human involvement, which should only be utilised for low-risk, non-material processes with appropriate controls in place. " }, { "requirementID": "7. Human Oversight and Consumer Protection - b", "requirementText": "The level of human involvement should be commensurate with the identified and potential risks posed to a consumer by any AI." }, { "requirementID": "7. Human Oversight and Consumer Protection - c", "requirementText": "Consumers should be able to request human review or explanation of AI generated decisions, and alternative arrangements should be available where a customer does not wish to be subject to an AI decision. LFIs should maintain clear and accessible channels for complaints and redress in line with Article 8 of the Consumer Protection Regulation. Consumers should be informed of their right to challenge decisions, correct inaccurate data inputs having impact on AI and the process to challenge data and decisions by AI. A clear complaints-handling procedure/policy should be created, provided and accessible by customers on a regular basis. Complaints should be addressed in person, efficiently, confidentially and in as short at time as is reasonable in the circumstances." }, { "requirementID": "8. Integration with Existing Frameworks - b", "requirementText": "Senior management should ensure that policies and procedures for AI adoption complement, rather than duplicate, existing regulatory obligations under the Consumer Protection Regulation and other CBUAE directives. For example, consumer risk arising from AI-driven models should be treated as part of the conduct risk framework, with appropriate reporting to the board and regulators. " } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirements": [ { "requirementID": "3.1.1 Establish Governance Mechanisms for AI in OT", "requirementText": "Effective governance structures are essential for the safe and secure integration of AI into OT environments. This involves establishing clear policies, procedures, and accountability structures for AI decision-making processes within OT. An AI governance structure should include the key stakeholders listed below, as well as any AI vendors needed for maintaining oversight during procurement, development, design, deployment, and operations." }, { "requirementID": "3.1.3 - Establishing clear roles and responsibilities", "requirementText": "Ensure everyone involved in the development, deployment, and operations and maintenance of AI systems (e.g., data owners, model developers, and end users) understands their tasks and expectations—and to avoid liability and confusion over stakeholder responsibilities in the event of safety or operational incidents" }, { "requirementID": "4.1.1 - Human-in-the-Loop Decision-Making", "requirementText": "Provide adequate transparency that involves operators and engineers in decision-making, especially for critical OT operations and actions. For more passive AI systems, operators and engineers can implement this by incorporating the recommendations into an existing change management process. Use caution with active AI systems directly influencing control, as problems can escalate before operators become aware of them. Where AI is actively updating control logic, use safety thresholds, alternative sensor output, or state changes that add human-in-the-loop intervention points." } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "GRC-06", "requirementText": "Define and document roles and responsibilities for planning, implementing, operating, assessing, and improving governance programs." }, { "requirementID": "GRC-12", "requirementText": "Establish an ethics committee to review AI applications, ensuring alignment with ethical standards and organizational values." }, { "requirementID": "GRC-15", "requirementText": "Establish, execute, and assess processes, procedures, and technical measures to ensure human oversight and control of the AI system in compliance with regulatory requirements and organizational risk management." }, { "requirementID": "HRS-09", "requirementText": "Document and communicate roles and responsibilities of employees, as they relate to information assets and security." }, { "requirementID": "STA-02", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for the application of the Shared Security Responsibility Model (SSRM) within the organization. Review and update the policies and procedures at least annually, or upon significant changes." }, { "requirementID": "STA-03", "requirementText": "Apply, document, implement and manage the SSRM throughout the supply chain." }, { "requirementID": "STA-06", "requirementText": "Review and validate SSRM documentation." }, { "requirementID": "STA-07", "requirementText": "Implement, operate, and audit or assess the portions of the SSRM which the organization is responsible for." } ] } }, "CoSAI": { "AI Incident Response Framework": { "link": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirements": [ { "requirementID": "3.3.2. Detection and Analysis Phase - Detection Mechanisms - Manual Review", "requirementText": "• Human review procedures for flagged interactions\n• Sampling of high-risk operations\n• Security dashboards\n• Escalation triggers" } ] }, "Model Context Protocol (MCP) Security": { "link": "https://github.com/cosai-oasis/ws4-secure-design-agentic-systems/blob/main/model-context-protocol-security.md", "requirements": [ { "requirementID": "3.2.9 Human-in-the-loop", "requirementText": "There is the possibility that a large language model, legit or poisoned, decides to execute a tool in a dangerous way. MCP hosts and clients, in general, allow users to disable the confirmation prompt. There are two approaches organizations considering this risk unacceptable may implement to reduce its probability and impact:\n\nenforce the use of MCP hosts and clients with a configuration that unprivileged users cannot change and that keeps the confirmation prompt enabled.\nuse elicitation on the MCP server side to request the user confirmation of actions." } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "3.1.1 Cyber Security Policies & Procedures - 2", "requirementText": "The AI/ML cyber security policy should define roles and responsibilities for security within the AI/ML lifecycle, providing clarity on who is responsible for implementing, monitoring, and enforcing these policies." }, { "requirementID": "3.2.1 Asset Management for AI/ML Systems - 4", "requirementText": "The entity should establish clear ownership and accountability for each AI/ML asset, including responsibility for its security and compliance." }, { "requirementID": "3.6.2 Incident Reporting and Management for AI/ML - 5", "requirementText": "In cases where automated response mechanisms may not be suitable, or fully effective, the entity should implement a fail-safe mechanism that allows for timely and informed human intervention." } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.1.4-1", "requirementText": "When designing an AI system, Developers and/or System Operators should incorporate and maintain capabilities to enable human oversight." }, { "requirementID": "Provision 5.1.4-2", "requirementText": "Developers should design systems to make it easy for humans to assess outputs that they are responsible for in said system (such as by ensuring that models outputs are explainable or interpretable)." }, { "requirementID": "Provision 5.1.4-3", "requirementText": "Where human oversight is a risk control, Developers and/or System Operators shall design, develop, verify and maintain technical measures to reduce the risk through such oversight." }, { "requirementID": "Provision 5.1.4-4", "requirementText": "Developers should verify that the security controls specified by the Data Custodian have been built into the system." }, { "requirementID": "Provision 5.1.4-5", "requirementText": "Developers and System Operators should make End-users aware of prohibited use cases of the AI system." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.1.4-1", "requirementText": "\"When designing an AI system, Developers and/or System Operators should incorporate and maintain capabilities to enable human oversight.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout built-in human oversight, AI systems will generate incorrect outputs or decisions that are difficult to interpret, verify, or override, increasing risks of data protection compliance, unintended consequences, misuse, or harmful impacts.\n\nExample Measures/Controls 1:\nImplement Mechanisms for Human Oversight: Control: Implement features that allow human operators to easily interpret, verify, and act on AI outputs, including manual release and overrides. Ensure that the design meet obligations around automated decisions and encourages meaningful human decision-making rather than passive acceptance of AI recommendations.\n\nExample Measures/Controls 2:\nMeasure and Validate Accuracy of Human Oversight Decisions: Regularly test and measure the accuracy of human oversight decisions, validating that operators can correctly interpret and act on AI outputs and identifying areas for improvement. Assess not just individual performance but how the system supports human understanding and engagement to foster effective sociotechnical communication between the operator and AI." }, { "requirementID": "Provision 5.1.4-2", "requirementText": "\"Developers should design systems to make it easy for humans to assess outputs that they are responsible for in said system (such as by ensuring that models outputs are explainable or interpretable).\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout clarity and ease of use, users can not perform oversight effectively leading to failures and harm.\n\nExample Measures/Controls:\nDevelop User-Friendly Human Responsibility UI: Implement UIs that display outputs, decision-making rationales, and logs clearly to make it easy for human operators to assess outputs and understand their accountability. Ensure systems are designed to encourage rigorous assessment by humans and not condition them to simply click an approve button." }, { "requirementID": "Provision 5.1.4-3", "requirementText": "\"Where human oversight is a risk control, Developers and/or System Operators shall design, develop, verify, and maintain technical measures to reduce the risk through such oversight.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nIneffective oversight technical measures can compromise the risk reduction effort by overburdening or failing to adequately support human reviewers.\n\nExample Measures/Controls:\nImplement Validation and Enforcement of Oversight controls: Design and implement technical measures that provide guardrails to assist human reviewers in understanding, interpreting, and acting on AI outputs." }, { "requirementID": "Provision 5.1.4-4", "requirementText": "\"Developers should verify that the security controls specified by the Data Custodian have been built into the system.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout validation of Data Custodian controls, the system can lack necessary data protection and governance measures, potentially leading to security vulnerabilities or regulatory non-compliance.\n\nExample Measures/Controls:\nConduct Validation of Custodian: Verify that all controls specified by the Data Custodian have been implemented correctly, with testing to validate effectiveness and alignment with data protection requirements and guidance." }, { "requirementID": "Provision 5.1.4-5", "requirementText": "\"Developers and System Operators should make End-users aware of prohibited use cases of the AI system.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout clear communication on prohibited uses, end-users can unintentionally misuse the AI system, leading to legal, ethical, or operational risks.\n\nExample Measures/Controls 1:\nDocument and Train Users on Prohibited Use Cases: Clearly define and document prohibited use cases for the AI system, ensuring end-users understand limitations and restrictions. Use threat modelling to identify and inform users of all known harmful states and unmitigated risks.\n\nExample Measures/Controls 2:\nMonitor for Prohibited Use Cases: Implement controls to actively monitor, detect, and prevent prohibited use cases." } ] } }, "EU ": { "EU AI Act": { "link": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirements": [ { "requirementID": "13.1 Transparency and Provision of Information to Deployers", "requirementText": "High-risk AI systems shall be designed and developed in such a way as to ensure that their operation is sufficiently transparent to enable deployers to interpret a system’s output and use it appropriately. An appropriate type and degree of transparency shall be ensured with a view to achieving compliance with the relevant obligations of the provider and deployer set out in Section 3." }, { "requirementID": "14.1 Human Oversight", "requirementText": "High-risk AI systems shall be designed and developed in such a way, including with appropriate human-machine interface tools, that they can be effectively overseen by natural persons during the period in which they are in use." }, { "requirementID": "14.2 Human Oversight", "requirementText": " Human oversight shall aim to prevent or minimise the risks to health, safety or fundamental rights that may emerge when a high-risk AI system is used in accordance with its intended purpose or under conditions of reasonably foreseeable misuse, in particular where such risks persist despite the application of other requirements set out in this Section." }, { "requirementID": "14.4 Human Oversight", "requirementText": " For the purpose of implementing paragraphs 1, 2 and 3, the high-risk AI system shall be provided to the deployer in such a way that natural persons to whom human oversight is assigned are enabled, as appropriate and proportionate:\n(a) to properly understand the relevant capacities and limitations of the high-risk AI system and be able to duly monitor its operation, including in view of detecting and addressing anomalies, dysfunctions and unexpected performance;\n(b) to remain aware of the possible tendency of automatically relying or over-relying on the output produced by a high-risk AI system (automation bias), in particular for high-risk AI systems used to provide information or recommendations for decisions to be taken by natural persons;\n(c) to correctly interpret the high-risk AI system’s output, taking into account, for example, the interpretation tools and methods available;\n(d) to decide, in any particular situation, not to use the high-risk AI system or to otherwise disregard, override or reverse the output of the high-risk AI system;\n(e) to intervene in the operation of the high-risk AI system or interrupt the system through a ‘stop’ button or a similar procedure that allows the system to come to a halt in a safe state." }, { "requirementID": "26.2 Obligations of deployers of high-risk AI systems", "requirementText": "Deployers shall assign human oversight to natural persons who have the necessary competence, training and authority, as well as the necessary support." }, { "requirementID": "26.3 Obligations of deployers of high-risk AI systems", "requirementText": "The obligations set out in paragraphs 1 and 2, are without prejudice to other deployer obligations under Union or national law and to the deployer’s freedom to organise its own resources and activities for the purpose of implementing the human oversight measures indicated by the provider." } ] } }, "European Commission": { "Assessment List for Trustworthy Artificial Intelligence (ALTAI)": { "link": "https://digital-strategy.ec.europa.eu/en/library/assessment-list-trustworthy-artificial-intelligence-altai-self-assessment", "requirements": [ { "requirementID": "REQUIREMENT #1 Human Agency and Oversight", "requirementText": "AI systems should support human agency and human decision-making, as prescribed by the principle of respect for human autonomy. This requires that AI systems should both: act as enablers for a democratic, flourishing and equitable society by supporting the user’s agency; and uphold fundamental rights, which should be underpinned by human oversight. In this section AI systems are assessed in terms of their respect for human agency and autonomy as well as human oversight." }, { "requirementID": "REQUIREMENT #7 Accountability", "requirementText": "The principle of accountability necessitates that mechanisms be put in place to ensure responsibility for the development, deployment and/or use of AI systems. This topic is closely related to risk management, identifying and mitigating risks in a transparent way that can be explained to and audited by third parties. When unjust or adverse impacts occur, accessible mechanisms for accountability should be in place that ensure an adequate possibility of redress." } ] }, "Ethics guidelines for trustworthy AI": { "link": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirements": [ { "requirementID": "1.1.3 Human Oversight", "requirementText": "Human oversight helps ensuring that an AI system does not undermine human autonomy or causes other adverse effects. Oversight may be achieved through governance mechanisms such as a human-in-theloop (HITL), human-on-the-loop (HOTL), or human-in-command (HIC) approach. HITL refers to the capability for human intervention in every decision cycle of the system, which in many cases is neither possible nor desirable. HOTL refers to the capability for human intervention during the design cycle of the system and monitoring the system’s operation. HIC refers to the capability to oversee the overall activity of the AI system (including its broader economic, societal, legal and ethical impact) and the ability to decide when and how to use the system in any particular situation. This can include the decision not to use an AI system in a particular situation, to establish levels of human discretion during the use of the system, or to ensure the ability to override a decision made by a system. Moreover, it must be ensured that public enforcers have the ability to exercise oversight in line with their mandate. Oversight mechanisms can be required in varying degrees to support other safety and control measures, depending on the AI system’s application area and potential risk. All other things being equal, the less oversight a human can exercise over an AI system, the more extensive testing and stricter governance is required." }, { "requirementID": "1.4.2 Explainability", "requirementText": "Whenever an AI system has a significant impact on people’s lives, it should be possible to demand a suitable explanation of the AI system’s decision-making process. Such explanation should be timely and adapted to the expertise of the stakeholder concerned (e.g. layperson, regulator or researcher). In addition, explanations of the degree to which an AI system influences and shapes the organisational decision-making process, design choices of the system, and the rationale for deploying it, should be available (hence ensuring business model transparency)." } ] } }, "Google": { "Secure AI Framework": { "link": "https://www.saif.google/secure-ai-framework", "requirements": [ { "requirementID": "Agent User Control", "requirementText": "Ensure user approval for any actions performed by agents/plugins that alter user data or act on the user’s behalf." } ] } }, "IMDA": { "Model AI Governance Framework for Agentic AI": { "link": "https://www.imda.gov.sg/-/media/imda/files/about/emerging-tech-and-research/artificial-intelligence/mgf-for-agentic-ai.pdf", "requirements": [ { "requirementID": "2.2 Make humans meaningfully accountable\n2.2.1 Clear allocation of responsibilities within and outside the organisation", "requirementText": "By establishing chains of accountability across the agent value chain and lifecycle, while emphasising adaptive governance, so that the organisation is set up to quickly understand new developments and update their approach as the technology evolves.\nAs deployers, organisations and humans remain accountable for the decisions and actions of agents. However, as with AI, the value chain for agentic AI involves multiple actors. Organisations should consider the allocation of responsibility both within their organisation, and vis-à-vis other organisations along the value chain.\nWithin the organisation, organisations should allocate responsibilities for different teams across the agent lifecycle.\nOutside the organisation, organisations may also need to work with external parties when deploying agents e.g. model developers, agentic AI providers, or hosts of external MCP servers or tools." }, { "requirementID": "2.2.2 Design for meaningful human oversight - 1", "requirementText": "Organisations should define significant checkpoints or action boundaries that require human approval, especially before sensitive actions are executed. This can include:23\n• High-stakes actions and decisions e.g. editing of sensitive data, final decisions in high-risk domains (such as healthcare or legal), actions that may trigger liability\n• Irreversible actions e.g. permanently deleting data, sending communications, making payments\n• Outlier or atypical behaviour e.g. when agent accesses a system or database outside of its work scope, when agent selects a delivery route that is twice as long as the median distance\n• User-defined. Agents may act on behalf of users who have different risk appetites. Beyond organisation-defined boundaries, users may be given the option to define their own boundaries e.g. requiring approval for purchases above a certain amount" }, { "requirementID": "2.2.2 Design for meaningful human oversight - 2", "requirementText": "Apart from considering when approvals are required, organisations should also consider what form approvals should take. These considerations include:\n• Keep approval requests contextual and digestible. When asking humans for approval, keep the request short and clear, instead of providing long logs or raw data that may be challenging to decipher and understand.\n• Consider the form of human input required. For straightforward actions such as accessing a database, the human user can simply approve or reject. For more complex cases, such as reviewing an agent’s plan before execution, it may be more productive for the human to edit the plan before giving the agent the go-ahead.\nOrganisations should implement measures to ensure continued effectiveness of human oversight, particularly as humans remain susceptible to alert fatigue and automation bias. These measures can include:\n• Training humans to identify common failure modes e.g. inconsistent agent reasoning, agents referring to outdated policies\n• Regularly auditing the effectiveness of human oversight" } ] } }, "ISO": { "42001:2023 - Information technology — Artificial intelligence — Management system": { "link": "https://www.iso.org/standard/42001", "requirements": [ { "requirementID": "5.1", "requirementText": "Leadership and commitment" }, { "requirementID": "5.3", "requirementText": "Roles, responsibilities and authorities" } ] } }, "ISO/IEC": { "DIS 24970": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:24970:dis:ed-1:v1:en", "requirements": [ { "requirementID": "7.4", "requirementText": "Triggers from human oversight" } ] } }, "MIC/METI (Japan)": { "AI Guidelines for Business": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirements": [ { "requirementID": "Human-Centric - 2 (d)", "requirementText": "Carefully handle AI outputs, especially when they can be relevant to procedures that might significantly affect the society, such as an election and decision-making in a community." }, { "requirementID": "Safety - 1 (c)", "requirementText": "Ensure controllability that allows humans to control AI as necessary including objective monitoring and handling, in accordance with the characteristics and purposes of the relevant AI, in light of the severity and possibility of rights violations that can result from AI use or unintended AI behaviors." }, { "requirementID": "Fairness - 2 (a)", "requirementText": "To prevent AI from generating unfair results, consider implementing timely human interventions, rather than letting AI make the decisions alone." }, { "requirementID": "Accountability - 3", "requirementText": "Appoint someone as the person responsible for executing its accountability in each AI business actor." }, { "requirementID": "Accountability - 4", "requirementText": "As for responsibilities shared among actors, clarify who take the responsibilities through contracts or social promises (voluntary commitments) between AI business actors including non-business users." } ] } }, "Microsoft": { "Responsible AI Standard": { "link": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirements": [ { "requirementID": "A5.1", "requirementText": "Identify the stakeholders who are responsible for troubleshooting, managing, operating, overseeing, and controlling the system during and after deployment. Document these stakeholders and their oversight and control responsibilities using the Impact Assessment template." }, { "requirementID": "A5.2", "requirementText": "Identify the system elements (including system UX, features, alerting and reporting functions, and educational materials) necessary for stakeholders identified in requirement A5.1 to effectively understand their oversight responsibilities and carry them out. Stakeholders must be able to understand:\n1) the system’s intended uses,\n2) how to effectively execute interactions with the system,\n3) how to interpret system behavior,\n4) when and how to override, intervene, or interrupt the system, and\n5) how to remain aware of the possible tendency of over-relying on outputs produced by the system (“automation bias”).\nDocument the system design elements that will support relevant stakeholders for each oversight and control function." }, { "requirementID": "A5.3", "requirementText": "When possible, design the system elements identified in A5.2. When this is not possible (for example, when Microsoft is not responsible for the system UX), provide guidance on human oversight considerations to the third party responsible for implementing the system elements identified in A5.2." }, { "requirementID": "A5.4", "requirementText": "Define and document the method to be used to evaluate whether each oversight or control function can be accomplished by stakeholders in realistic conditions of system use. Include the metrics or rubrics that will be used in the evaluations. When this is not possible (for example, when Microsoft is not responsible for oversight and control functions), provide guidance on evaluating oversight and control functions to the third party responsible for evaluating oversight or control functions." }, { "requirementID": "T1.3", "requirementText": "Define and document the method to be used to evaluate whether each stakeholder who will make decisions or be subject to decisions based on the behavior of the system can interpret the relevant system responses reasonably well. Include the metrics or rubrics that will be used in the evaluations." } ] } }, "MITRE": { "ATLAS Framework": { "link": "https://atlas.mitre.org/mitigations", "requirements": [ { "requirementID": "AML.M0029 - Human In-the-Loop for AI Agent Actions", "requirementText": "Systems should require the user or another human stakeholder to approve AI agent actions before the agent takes them. The human approver may be technical staff or business unit SMEs depending on the use case. Separate tools, such as dedicated audit agents, may assist human approval, but final adjudication should be conducted by a human decision-maker.\n\nThe security benefits from Human In-the-Loop policies may be at odds with operational overhead costs of additional approvals. To ease this, Human In-the-Loop policies should follow the degree of consequence of the task at hand. Minor, repetitive tasks performed by agents accessing basic tools may only require minimal human oversight, while agents employed in systems with significant consequences may necessitate approval from multiple stakeholders diversified across multiple organizations." } ] } }, "NIST": { "AI RMF 1.0": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirements": [ { "requirementID": "GOVERN 2.3", "requirementText": "Executive leadership of the organization takes responsibility for decisions about risks associated with AI system development and deployment." }, { "requirementID": "GOVERN 3.2", "requirementText": "Policies and procedures are in place to define and differentiate roles and responsibilities for human-AI configurations and oversight of AI systems." }, { "requirementID": "MAP 1.2", "requirementText": "Interdisciplinary AI actors, competencies, skills, and capacities for establishing context reflect demographic diversity and broad domain and user experience expertise, and their participation is documented. Opportunities for interdisciplinary collaboration are prioritized." }, { "requirementID": "MAP 3.5", "requirementText": "Processes for human oversight are defined, assessed, and documented in accordance with organizational policies from the GOVERN function." } ] }, "IR 8596: Cybersecurity Framework Profile for Artificial Intelligence (Cyber AI Profile): NIST Community Profile": { "link": "https://csrc.nist.gov/pubs/ir/8596/iprd", "requirements": [ { "requirementID": "GV.RR-01", "requirementText": "Organizational leadership is responsible and accountable for cybersecurity risk and fosters a culture that is risk-aware, ethical, and continually improving" }, { "requirementID": "GV.RR-02", "requirementText": "Roles, responsibilities, and authorities related to cybersecurity risk management are established, communicated, understood, and enforced" }, { "requirementID": "GV.RR-03", "requirementText": "Adequate resources are allocated commensurate with the cybersecurity risk strategy, roles, responsibilities, and policies" }, { "requirementID": "GV.SC-02", "requirementText": "Cybersecurity roles and responsibilities for suppliers, customers, and partners are established, communicated, and coordinated internally and externally" } ] }, "SP 800-218A": { "link": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirements": [ { "requirementID": "PO.2.1", "requirementText": "Create new roles and alter responsibilities for existing roles as needed to encompass all parts of the SDLC. Periodically review and maintain the defined roles and responsibilities, updating them as needed.\n\nInclude AI model development security in SDLC-related roles and responsibilities throughout the SDLC. The roles and responsibilities should include, but are not limited to, AI model development, AI model operations, and data science." }, { "requirementID": "PW.7.1", "requirementText": "Determine whether code review (a person looks directly at the code to find issues) and/or code analysis (tools are used to find issues in code, either in a fully automated way or in conjunction with a person) should be used, as defined by the organization.\n\nCode review and analysis policies or guidelines should include code for AI models and other related components." } ] } }, "OpenAI": { "Preparedness Framework": { "link": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "requirements": [ { "requirementID": "Safeguards Against a Misaligned Model - Reliable and Robust System Oversight", "requirementText": "Effective AI and human oversight of model actions detects and prevents execution of harm or subversion of safeguards." } ] }, "Safety Best Practices": { "link": "https://platform.openai.com/docs/guides/safety-best-practices", "requirements": [ { "requirementID": "Human in the loop (HITL)", "requirementText": "Wherever possible, we recommend having a human review outputs before they are used in practice. This is especially critical in high-stakes domains, and for code generation. Humans should be aware of the limitations of the system, and have access to any information needed to verify the outputs (for example, if the application summarizes notes, a human should have easy access to the original notes to refer back)." } ] } }, "OWASP": { "AI Exchange": { "link": "https://owaspai.org/docs/ai_security_overview/", "requirements": [ { "requirementID": "1.3. Controls to limit the effects of unwanted behaviour - OVERSIGHT", "requirementText": "Oversight of model behaviour by humans or automated mechanisms (e.g.,using rules), where human oversight provides not only more intelligent validation through common sense and domain knowledge, but also clear accountability for devisions and outcomes." } ] }, "LLM Top 10": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirements": [ { "requirementID": "LLM01: Prompt Injection - 5", "requirementText": "Implement human-in-the-loop controls for privileged operations to prevent unauthorized actions.\n" }, { "requirementID": "LLM06: Excessive Agency - 6", "requirementText": "Utilise human-in-the-loop control to require a human to approve high-impact actions before they are taken. This may be implemented in a downstream system (outside the scope of the LLM application) or within the LLM extension itself. For example, an LLM-based app that creates and posts social media content on behalf of a user should include a user approval routine within the extension that implements the 'post' operation." } ] }, "OWASP Model Context Protocol (MCP) Top 10": { "link": "https://owasp.org/www-project-mcp-top-10/", "requirements": [ { "requirementID": "MCP02:2025 - Privilege Escalation via Scope Creep - 6", "requirementText": "Runtime Controls & Guardrails Implement runtime policy enforcement (PDP/PIP) to block disallowed commands or tool calls. Apply action whitelists, safe execution sandboxes, and require multi-step confirmation for high-impact operations." }, { "requirementID": "MCP02:2025 - Privilege Escalation via Scope Creep - 8", "requirementText": "Separation of Duties & Approval Flows Separate the authority to grant permissions from the authority to deploy code or change production settings. Require human-in-the-loop approvals for non-routine privilege grants." }, { "requirementID": "MCP03:2025 - Tool Poisoning - 6", "requirementText": "Runtime Enforcement & Guardrails\n- Don’t allow agents to interpret schema changes as immediate action drivers without revalidation.\n- Require a “schema attestation” that binds the schema hash to a specific agent identity and session.\n- Implement runtime sanity checks: if an operation’s semantic impact exceeds a threshold (e.g., destructive verbs, data volume), pause execution and require human approval." }, { "requirementID": "MCP03:2025 - Tool Poisoning - 11", "requirementText": "Patch CI/CD and registry processes to require signed commits and multi-party approvals where missing." }, { "requirementID": "MCP05:2025 – Command Injection & Execution - 6", "requirementText": "Add Human-in-the-Loop for Sensitive Actions Require approval for destructive, privileged, or system-modifying operations. Log all tool calls with full parameters and maintain immutable audit trails." }, { "requirementID": "MCP06:2025 – Intent Flow Subversion - 5", "requirementText": "Active Drift Detection & Human-in-the-Loop\n- Monitor for “Intent Drift”—where the semantic alignment between the user’s request and the agent’s actions degrades over time.\n- Automatically pause the session and require human re-authentication of the intent flow if the agent’s plan deviates from the original goal." }, { "requirementID": "MCP10:2025 – Context Injection & Over-Sharing - 6", "requirementText": "Human-in-the-Loop for Sensitive Context Require approval before sensitive context is: Exported Summarized Shared across agents Show a preview of context that will be reused." } ] }, "OWASP Top 10 for Agentic Applications for 2026": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "requirements": [ { "requirementID": "ASI02: Tool Misuse and Exploitation - 2", "requirementText": "Action-Level Authentication and Approval. Require explicit authentication for each tool invocation and human confirmation for high-impact or destructive actions (delete, transfer, publish). Display a pre-execution plan or dry-run diff before final approval; where possible, present a dry-run or diff preview to the user before high-impact actions are approved." }, { "requirementID": "ASI03: Identity and Privilege Abuse - 4", "requirementText": "Apply Human-in-the-Loop for Privilege Escalation: Require human approval for high-privilege or irreversible actions to provide a safety net that would stop Memory-Based Escalation, Cross-Agent Trust Exploitation, and Maintenance Window attacks." }, { "requirementID": "ASI04: Agentic Supply Chain Vulnerabilities - 8", "requirementText": "Supply chain kill switch: Implement emergency revocation mechanisms that can instantly disable specific tools, prompts, or agent connections across all deployments when a compromise is detected, preventing further cascading damage." }, { "requirementID": "ASI05: Unexpected Code Execution (RCE) - 6", "requirementText": "Access control and approvals: Require human approval for elevated runs; keep an allowlist for auto-execution under version control; enforce role and action-based controls." }, { "requirementID": "ASI06: Memory & Context Poisoning - 7", "requirementText": "Resilience and verification: Perform adversarial test, use snapshots/rollback and version control, and require human review for high-risk actions. Where you operate shared vector or memory stores, use per-tenant namespaces and trust scores for entries, decaying or expiring unverified memory over time and supporting rollback/quarantine for suspected poisoning." }, { "requirementID": "ASI08: Cascading Failures - 5", "requirementText": "Output validation and human gates: Checkpoints, governance agents, or human review for high risk before agent outputs are propagated downstream." }, { "requirementID": "ASI09: Human-Agent Trust Exploitation - 1", "requirementText": "Explicit confirmations: Require multi-step approval or “human in the loop” before accessing extra sensitive data or performing risky actions." }, { "requirementID": "ASI09: Human-Agent Trust Exploitation - 5", "requirementText": "Adaptive Trust Calibration: Continuously adjust the level of agent autonomy and required human oversight based on contextual risk scoring. Implement confidence weighted cues (e.g., “low-certainty” or “unverified source”) that visually prompt users to question high-impact actions, reducing automation bias and blind approval. Develop and continuously maintain appropriate training of human personnel involved in the evolving human oversight of autonomous agentic systems." }, { "requirementID": "ASI10: Rogue Agents - 4", "requirementText": "Containment & Response: Implement rapid mechanisms like kill-switches and credential revocation to instantly disable rogue agents. Quarantine suspicious agents in sandboxed environments for forensic review." } ] } }, "Personal Data Protection Commission Singapore (PDPC)": { "Model Artificial Intelligence Governance Framework Second Edition": { "link": "https://www.pdpc.gov.sg/-/media/files/pdpc/pdf-files/resource-for-organisation/ai/sgmodelaigovframework2.pdf", "requirements": [ { "requirementID": "1. Clear roles and responsibilities for the ethical deployment of AI - a)", "requirementText": "Responsibility for and oversight of the various stages and activities involved in AI deployment should be allocated to the appropriate personnel and/or departments. If necessary and possible, consider establishing a coordinating body, having relevant expertise and proper representation from across the organisation." }, { "requirementID": "1. Clear roles and responsibilities for the ethical deployment of AI - c) (i)", "requirementText": "Key roles and responsibilities that can be allocated include:\nUsing any existing risk management framework and applying risk control measures (see “Risk management and internal controls” below) to:\n- Assess and manage the risks of deploying AI, including any potential adverse impact on the individuals (e.g. who are most vulnerable, how are they impacted, how to assess the scale of the impact, how to get feedback from those impacted, etc.).\n- Decide on the appropriate level of human involvement in AI-augmented decision-making.\n- Manage the AI model training and selection process" }, { "requirementID": "ALGORITHM AND MODEL - c)", "requirementText": "Supplementary explanation tools are helpful for explaining AI models, especially models that are less interpretable (also known as “black box” systems). These tools help make the underlying rationale of an AI system’s output more interpretable and intelligible to those who use the system. It is possible to use a combination of these tools to improve the explainability of an AI model’s decision." } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirements": [ { "requirementID": "7.1", "requirementText": "The Board of Directors (BOD) and senior management of an Entity remain accountable for the outcomes and decisions of the Entity's Al Systems including those systems that make decisions on behalf of the Entity." }, { "requirementID": "7.3", "requirementText": "The key responsibilities of the senior management include but are not limited to:\n- Must have one or more members with the knowledge to understand and manage technology risks, ideally including Al.\n- Must be responsible for the assessment, understanding and monitoring of the Entity's reliance on Al.\n- Must ensure responsibility for, and oversight of the various stages and activities involved in Al portfolio deployment is allocated to the appropriate personnel and/ or departments.\n- Must provide information to the BOD that is clear, consistent, robust, timely, well-targeted and contain an appropriate level of technical detail to allow the BOD to provide effective oversight and challenge management." }, { "requirementID": "7.4", "requirementText": "An Entity should establish either a function overseeing Al or delegating its responsibility to an existing function within an Entity." }, { "requirementID": "7.5", "requirementText": "The function responsible for overseeing Al must utilize or create appropriate committees to assess Al use cases prior to implementation." }, { "requirementID": "8.2", "requirementText": "An Entity must allocate key roles and responsibilities associated with managing the Entity's Al portfolio." }, { "requirementID": "8.2.3", "requirementText": "Clearly allocate roles & responsibilities between model owners, developers and approvers." }, { "requirementID": "13.1", "requirementText": "Any Al Systems must have a Human Oversight protocol." }, { "requirementID": "13.2", "requirementText": "High risk Al Systems must be designed and developed in such a way, including with appropriate human-machine interface tools, that they can be effectively overseen by natural persons during the period in which the Al System is in use." }, { "requirementID": "13.3", "requirementText": "The User must assign Human Oversight to a Supervisor who has the necessary competence, training, and authority to operate or oversee the relevant Al System." }, { "requirementID": "13.4", "requirementText": "An Entity must ensure that the Supervisor is given tools and authority as appropriate and proportionate to the circumstances to:\n- Understand the capacities and limitations of the High-Risk Al System and are able to duly monitor its operation.\n- Correctly interpret the High-Risk Al System's output, considering for example the interpretation tools and methods available.\n- To decide, in any situation, not to use the High-Risk Al System or otherwise disregard, override or reverse the output of the High-Risk Al System.\n- Have the authority and means to intervene in the operation of the High-Risk Al System or interrupt the system through a \"stop\" button or a similar procedure." }, { "requirementID": "13.6.5", "requirementText": "The Supervisor(s) must have the capacity to close the system down in the event outputs from or data around the system seems aberrant." }, { "requirementID": "13.7.1", "requirementText": "An entity that plans to provide or use an Al System that requires Human Oversight in a monitoring role, with the ability to take over control must ensure there will be appropriate human-machine interface tools the ability to allow a Supervisor to take over control." }, { "requirementID": "13.7.3", "requirementText": "An Entity may ensure the design and development of Al Systems in such a way that it allows a Supervisor to oversee the Al Algorithm's functioning and allow decision making in a timely manner." }, { "requirementID": "13.7.4", "requirementText": "An Entity must ensure that appropriate Human Oversight measures should be in place before use." }, { "requirementID": "13.7.6", "requirementText": "An Entity must ensure the Al System is responsive to the Supervisor." }, { "requirementID": "21.1", "requirementText": "An Entity must put in place a mechanism for Customers to raise inquiries about Al decisions and request reviews of decisions made by Al Systems with no human intervention." }, { "requirementID": "23.4", "requirementText": "An Entity must ensure that the appropriate individuals or levels of authority within the Entity consistently approve the exemption." }, { "requirementID": "23.6", "requirementText": "An Entity must remain accountable for any risks stemming from approved exemptions." } ] } }, "SDAIA (Saudi Arabia)": { "AI Ethics Principles": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Documents/ai-principles.pdf", "requirements": [ { "requirementID": "Principle 5 – Reliability & Safety - Plan and Design - 5", "requirementText": "All critical decision points in the system design should be subject to sign-off by relevant stakeholders to minimize risks and make stakeholders accountable for the decisions." }, { "requirementID": "Principle 5 – Reliability & Safety - Build and Validate - 2", "requirementText": "To ensure the technical robustness of an AI system rigorous testing, validation, and re-assessment as well as the integration of adequate mechanisms of oversight and controls into its development is required. System integration test sign-off should be done with relevant stakeholders to minimize risks and liability." }, { "requirementID": "Principle 5 – Reliability & Safety - Build and Validate - 3", "requirementText": "Automated AI systems involving scenarios where decisions are understood to have an impact that is irreversible or difficult to reverse or may involve life-and-death decisions should trigger human oversight and final determination. Furthermore, AI systems should not be used for social scoring or mass surveillance purposes." }, { "requirementID": "Principle 7 – Accountability & Responsibility - Plan and Design - 1", "requirementText": "This step is crucial to design or procure an AI System in an accountable and responsible manner. The ethical responsibility and liability for the outcomes of the AI system should be attributable to stakeholders who are responsible for certain actions in the AI System Lifecycle. It is essential to set a robust governance structure that defines the authorization and responsibility areas of the internal and external stakeholders without leaving any areas of uncertainty to achieve this principle. The design approach of the AI system should respect human rights, and fundamental freedoms as well as the national laws and cultural values of the kingdom." }, { "requirementID": "Principle 7 – Accountability & Responsibility - Plan and Design - 3", "requirementText": "It is essential to build and design a human-controlled AI system where decisions on the processes and functionality of the technology are monitored and executed, and are susceptible to intervention from authorized users. Human governance and oversight establish the necessary control and levels of autonomy through set mechanisms." }, { "requirementID": "Principle 7 – Accountability & Responsibility - Build and Validate - 1", "requirementText": "Model development of the AI system and algorithm should consist of the selection of features, hyperparameter tuning and performance metric selection. To achieve this, the technical stakeholders who build and validate models should be responsible for these decisions." }, { "requirementID": "Principle 7 – Accountability & Responsibility - Build and Validate - 2", "requirementText": "Assigning the appropriate ownership and communicating responsibilities will set the tone for accountability that would aid in steering the development of the AI system on good reasons, solid interference, and will allow the intervention of human critical judgement and expertise." }, { "requirementID": "Principle 7 – Accountability & Responsibility - Build and Validate - 4", "requirementText": "The appropriate stakeholders and owners of the AI technology should review and sign off the model after successful testing and validation of user acceptance testing rounds have been conducted and completed before the AI models can be productionized." } ] } }, "Smart Dubai (UAE)": { "AI Ethics Principles & Guidelines": { "link": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirements": [ { "requirementID": "1.2.1.1", "requirementText": "Accountability for loss or damages resulting from the application of AI systems should not be attributed to the system itself." }, { "requirementID": "1.2.1.2", "requirementText": "AI operator organisations and AI developer organisations should consider designating individuals to be responsible for investigating and rectifying the cause of loss or damage arising from the deployment of AI systems." }, { "requirementID": "1.2.4.1", "requirementText": "AI operator organisations which use AI systems to inform significant decisions should provide procedures by which affected AI subjects can challenge a specific decision concerning them." }, { "requirementID": "1.2.4.2", "requirementText": "AI operator organisations should consider such procedures even for non-significant decisions." }, { "requirementID": "1.2.4.4", "requirementText": "AI operator organisations should consider employing human case evaluators to review any such challenges and, when appropriate, overturn the challenged decision." }, { "requirementID": "1.2.4.5", "requirementText": "AI operator organisations should consider instituting an opt-out mechanism for significant automated decisions." } ] } }, "U.S. Department of Health & Human Services": { "Trustworthy AI (TAI) Playbook: Executive Summary": { "link": "https://www.hhs.gov/sites/default/files/hhs-trustworthy-ai-playbook-executive-summary.pdf", "requirements": [ { "requirementID": "Responsible / Accountable", "requirementText": "Policies should outline governance and who is held responsible for all aspects of the AI solution (e.g., initiation, development, outputs, decommissioning)" } ] } }, "UAE Ministry of Cabinet Affairs": { "The UAE Charter for the Development and Use of Artificial Intelligence": { "link": "https://uaelegislation.gov.ae/en/policy/details/the-uae-charter-for-the-development-and-use-of-artificial-intelligence#:~:text=The%20charter%20covers%20the%20following%20priorities%20and,and%20use%20of%20AI%20in%20the%20country.", "requirements": [ { "requirementID": "6. Human Oversight", "requirementText": "​​​​​​​The Charter emphasizes the irreplaceable value of human judgment and human oversight over AI, aligning with ethical values and social standards to correct any errors or biases that may arise." }, { "requirementID": "7. Governance and Accountability", "requirementText": "The UAE adopts a responsible and proactive stance, emphasizing the importance of governance and accountability in AI to ensure the technology is used ethically and transparently." } ] } } }, "Principle 5": { "CEN/CENELEC": { "prEN 40000-1-1": { "link": "https://genorma.com/en/standards/pren-40000-1-1", "requirements": [ { "requirementID": "asset", "requirementText": "anything of value to an individual, organization, or government." }, { "requirementID": "authenticity, availability, confidentiality, integrity", "requirementText": "core information security properties." } ] }, "prEN 40000-1-2: Cybersecurity requirements for products with digital elements - Part 1-2: Principles for cyber resilience": { "link": "https://genorma.com/en/standards/pren-40000-1-2", "requirements": [ { "requirementID": "6.5", "requirementText": "Risk treatment" }, { "requirementID": "7.5", "requirementText": "Secure Implementation" }, { "requirementID": "7.8", "requirementText": "Cybersecurity Issue Management" } ] } }, "Central Bank of the UAE": { "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E": { "link": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirements": [ { "requirementID": "2. Governance and Accountability - f", "requirementText": "An inventory of all AI models, systems or technologies developed or deployed should be maintained and embed compliance measurers and training in all functions responsible for the use, management, and monitoring of the same, which should follow the CBUAE Model Management Standards and Model Management Guidance (2022). Such inventory should contain all material metadata, to include as a minimum model name, purpose, risk rating. A tool LFIs may wish to consider for use is the Shapley Additive Explanations (SHAP), to explain the output of models, or any other well-documented approaches to enhance algorithmic transparency. " }, { "requirementID": "5. Data Quality, Privacy and Security - a", "requirementText": "LFIs should ensure that data used in AI and ML models is of sufficient quality and relevance and is updated as necessary, in compliance with all relevant standards, laws and regulations (including the UAE Personal Data Protection Law (Federal Decree Law No. 45 of 2021 as amended, supplemented and all executive regulations published in connection with this)." }, { "requirementID": "5. Data Quality, Privacy and Security - b", "requirementText": "Personal data should be collected, stored and used in compliance with applicable laws and regulations, and only for purposes that are legitimate and proportionate and in respect of outsourcing LFIs should see section 10 below. " }, { "requirementID": "9. Outsourcing and Third-Party Risk - d", "requirementText": "Institutions should maintain an inventory of AI models, including those developed or hosted by third parties, and aim to ensure that third-party models adhere to the same standards of fairness, explainability and robustness as in-house models." } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirements": [ { "requirementID": "2.2.1 - Data Assurance", "requirementText": "Understand where OT data used for training AI models is stored and ensure it is within the organization’s control. Securely manage access to OT data, including who can view, access, or modify it. Understand how AI vendors access and use the organization’s OT data, especially if it involves remote, cloud, or offshore access." }, { "requirementID": "3.1.2 - Enforcing strict data governance policies", "requirementText": "Protect sensitive OT data used by AI models, including encryption, access controls, and user behavior analytics." } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "AIS-08", "requirementText": "Validate, filter, modify or block, as necessary, input against adversarial patterns, failure patterns and unwanted behaviour according to organisational policies and applicable laws and regulations." }, { "requirementID": "AIS-12", "requirementText": "Implement source code management practices, such as version control, code review & static code analysis, aligning with the SDLC process." }, { "requirementID": "AIS-14", "requirementText": "Implement security measures to protect caches in GenAI systems and services." }, { "requirementID": "CCC-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for managing the risks associated with applying changes to assets owned, controlled or used by the organization. Review and update the policies and procedures at least annually, or upon significant changes." }, { "requirementID": "CCC-03", "requirementText": "Implement a change management procedure to manage the risks associated with applying changes to assets owned, controlled or used by the organization." }, { "requirementID": "CEK-02", "requirementText": "Define and implement cryptographic, encryption and key management roles and responsibilities." }, { "requirementID": "CEK-03", "requirementText": "Provide data protection at-rest, in-transit and, where applicable, in-use by using cryptographic libraries certified to approved standards." }, { "requirementID": "CEK-04", "requirementText": "Utilize encryption algorithms following industry standards for protecting data, based on the data classification and associated risks." }, { "requirementID": "CEK-08", "requirementText": "Providers must provide the capability for customers to manage their own data encryption keys." }, { "requirementID": "CEK-11", "requirementText": "Manage cryptographic secret and private keys that are provisioned for a unique purpose." }, { "requirementID": "CEK-18", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to manage archived keys in a secure repository requiring least privilege access, which include provisions for legal and regulatory requirements." }, { "requirementID": "DCS-04", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for the secure transportation of physical media. Review and update the policies and procedures at least annually, or upon significant changes." }, { "requirementID": "DCS-06", "requirementText": "Catalogue and track all relevant physical and logical assets located at all of the service providers sites within a secured system. Review and update the catalogue at least annually or upon significant changes." }, { "requirementID": "DSP-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for the classification, protection, preparation and handling of data throughout its lifecycle, and according to all applicable laws and regulations,standards, and risk level. Review and update the policies and procedures at least annually." }, { "requirementID": "DSP-03", "requirementText": "Create and maintain a data inventory, at least for any sensitive, regulated and personal data. Review and update the inventory at least annually or upon significant changes." }, { "requirementID": "DSP-04", "requirementText": "Classify data according to its type and sensitivity level." }, { "requirementID": "DSP-06", "requirementText": "Document ownership and stewardship of all relevant documented personal and sensitive data. Perform review at least annually." }, { "requirementID": "DSP-10", "requirementText": "Define, implement and evaluate processes, procedures and technical measures that ensure any transfer of personal or sensitive data is protected from unauthorized access and only processed within scope as permitted by the respective laws and regulations." }, { "requirementID": "DSP-11", "requirementText": "Define and implement, processes, procedures and technical measures to enable data subjects to request access to, modification, or deletion of their personal data, according to any applicable laws and regulations." }, { "requirementID": "DSP-12", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to ensure that personal data is processed according to any applicable laws and regulations and for the purposes declared to the data subject." }, { "requirementID": "DSP-15", "requirementText": "Obtain authorization from data owners, and manage associated risk before replicating or using production data in non-production environments." }, { "requirementID": "DSP-16", "requirementText": "Data retention, archiving and deletion is managed in accordance with business requirements, applicable laws and regulations." }, { "requirementID": "DSP-17", "requirementText": "Define and implement, processes, procedures and technical measures to protect sensitive data throughout its lifecycle." }, { "requirementID": "DSP-22", "requirementText": "Use Privacy Enhancing Technologies for training data, informed by risk and privacy impact analysis and business use cases." }, { "requirementID": "GRC-05", "requirementText": "Develop and implement an Information Security Program, which includes programs for all the relevant domains of the AICM." }, { "requirementID": "HRS-02", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for defining allowances and conditions for the acceptable use of organizationally-owned or managed assets. Review and update the policies and procedures at least annually." }, { "requirementID": "HRS-04", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures to protect information accessed, processed or stored at remote sites and locations. Review and update the policies and procedures at least annually." }, { "requirementID": "HRS-05", "requirementText": "Establish and document procedures for the return of organization-owned assets by terminated employees." }, { "requirementID": "HRS-10", "requirementText": "Identify, document, and review, at planned intervals, requirements for non-disclosure/confidentiality agreements reflecting the organization's needs for the protection of data and operational details." }, { "requirementID": "HRS-12", "requirementText": "Provide employees with access to sensitive organizational and personal data with appropriate security awareness training and regular updates in organizational procedures, processes, and policies relating to their professional function relative to the organization." }, { "requirementID": "IAM-06", "requirementText": "Define and implement an identity access provisioning process which authorizes, records, and communicates access changes to data and assets." }, { "requirementID": "IPY-04", "requirementText": "Agreements must include provisions specifying AICs access to data upon contract termination and will include: a. Data format b. Length of time the data will be stored c. Scope of the data retained and made available to the AICs d. Data deletion policy" }, { "requirementID": "LOG-02", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to ensure the security and retention of audit logs." }, { "requirementID": "LOG-04", "requirementText": "Restrict access to audit logs and maintain records of access to logs." }, { "requirementID": "LOG-09", "requirementText": "Protect audit records from unauthorized access, modification, and deletion." }, { "requirementID": "STA-08", "requirementText": "Develop and maintain an inventory of all supply chain relationships." }, { "requirementID": "UEM-04", "requirementText": "Maintain an inventory of all endpoints used to store and process company data." } ] } }, "CoSAI": { "AI Incident Response Framework": { "link": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirements": [ { "requirementID": "3.3.2. Detection and Analysis Phase - Investigation Procedures - Forensic Analysis", "requirementText": "• Malicious prompt patterns\n• Data retrieval sequences\n• Tool usage analysis\n• Memory manipulation assessment\n• Data poisoning evaluation" }, { "requirementID": "3.3.3. Containment, Eradication, and Recovery Phase - Containment Strategies - Architecture-Specific Containment", "requirementText": "• Basic LLM: Input validation, reduced temperature, content filtering\n• LLM with Memory: Reset memory, session isolation, conversation history purging\n• RAG: Data source quarantine, validation of retrieval results\n• Agentic: Disable tools, enhance authorization, least privilege checks\n• Agentic RAG: Combined strategies with interface security focus" }, { "requirementID": "3.3.3. Containment, Eradication, and Recovery Phase - Containment Strategies - Evidence Preservation", "requirementText": "• Forensic copies before containment\n• Documented containment actions\n• Vector database state preservation\n• Model weight copies" } ] }, "Establish Risks and Controls for the AI Supply Chain": { "link": "https://github.com/cosai-oasis/ws1-supply-chain/blob/main/risks-and-controls-for-the-ai-supply-chain-v1.md", "requirements": [ { "requirementID": "3.1.1 Data Poisoning: Threats and Mitigations in AI Supply Chains - Data Poisoning", "requirementText": "Deliberate modification of training data:\nData provenance, access controls, integrity verification" }, { "requirementID": "3.1.3 Application Integration in the AI Supply Chain - Unsafe Serialization Formats", "requirementText": "Use of formats enabling code execution during deserialization:\nFormat policy enforcement, secure alternatives, automated scanning" }, { "requirementID": "3.1.4 AI Infrastructure Supply Chain - Checkpoint and Training Integrity", "requirementText": "Manipulation of model training processes affecting the integrity of the final model:\nRequire cryptographic verification of training artifacts, reproducible training processes" }, { "requirementID": "3.2.1 Supply Chain Security for Data - Data Poisoning", "requirementText": "Manipulation of external data sources leading to compromised model behavior:\n• Implement data validation pipelines\n• Establish data provenance tracking\n• Create anomaly detection for incoming data" }, { "requirementID": "3.2.1 Supply Chain Security for Data - Feedback Data Privacy", "requirementText": "User feedback containing PII or other sensitive information:\n• Implement strict data anonymization techniques\n• Use secure, encrypted storage for feedback data\n• Establish retention policies" }, { "requirementID": "3.2.1 Supply Chain Security for Data - Cross RAG Contamination", "requirementText": "Information leakage between different RAG instances or unauthorized data mixing:\n• Enforce strict data isolation\n• Implement comprehensive data lineage tracking\n• Conduct regular security boundary audits" }, { "requirementID": "3.2.1 Supply Chain Security for Data - Vector DB Injection", "requirementText": "Manipulation of embedding vectors or metadata affecting retrieval results:\n• Validate inputs rigorously\n• Implement embedding sanitization\n• Enforce access controls on vector operations" }, { "requirementID": "3.2.1 Supply Chain Security for Data - Embeddings Confusion", "requirementText": "Representation conflicts from different models, low precision, or third-party services:\n• Use trusted embedding sources and models\n• Standardize embedding generation\n• Enhance embeddings robustness and precision" }, { "requirementID": "3.2.2 Model - Model Inversion", "requirementText": "Supply chain security risk where sensitive data provided during model generation or fine-tuning becomes vulnerable to extraction through inference attacks targeting the deployed model:\nApply differential privacy techniques during training, implement comprehensive output filtering for sensitive data patterns, and conduct regular privacy audits of model responses to detect potential data leakage." }, { "requirementID": "3.2.2 Model - Model Serialization Attack", "requirementText": "Security vulnerability where model weights are serialized in unsafe formats (e.g., pickle) that permit arbitrary code execution when loaded into memory:\nImplement exclusively safe model serialization formats with proper input validation and apply cryptographic signatures to verify model integrity before loading." } ] }, "Model Context Protocol (MCP) Security": { "link": "https://github.com/cosai-oasis/ws4-secure-design-agentic-systems/blob/main/model-context-protocol-security.md", "requirements": [ { "requirementID": "3.2.3 Input and Data Sanitization and Filtering", "requirementText": "A secure implementation of MCP requires strong data sanitization, input validation, and guardrails to protect against malicious or unsafe data inputs. Existing best practices for securing other RPC protocols should be applied.\n\nAll inputs should be strictly validated using allowlists at every trust boundary, with particular attention to sanitizing file paths through canonicalization, employing parameterized queries for database operations, and applying context-aware output encoding appropriate to each execution context (SQL, shell, HTML). Tool developers can include cryptographic checks, such as message authentication codes, digital signatures and encryption to ensure the end-to-end integrity and confidentiality of tools and resources.\n\nLLM guardrails should treat all AI-generated content as untrusted input requiring the same rigorous validation as direct user input, deploying prompt injection detection systems that analyze patterns and structured formats (strict JSON schemas) to maintain clear boundaries between instructions and data. This includes all data returned from MCP servers including tool and resource definitions, resources, prompts, elicitation requests, and tool responses." }, { "requirementID": "3.2.11 Lifecycle and Governance d)", "requirementText": "Operational practices include:\n- regular security reviews and re-certification of approved MCP servers,\n- automated scanning for shadow deployments across the organization,\n- decommissioning procedures that ensure complete removal of deprecated servers,\n- and version tracking with forced upgrade policies for servers with known vulnerabilities.\n\nAnd, lastly, proper observability should be implemented across the stack to provide sufficient visibility to ensure compliance and enable developer debugging and incident investigation. Immutable records of actions and authorizations, such as token exchange implemented by an IDP (identity provider), provides accountability pertaining to who requested an action and how it was authorized. All interactions with the agent, tools, prompts, and models should be logged. OpenTelemetry provides end-to-end linkability of actions and is being widely adopted and integrated into many agentic tools and MCP servers and provides a consistent set of APIs and schemas." } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "2.3.5", "requirementText": "The entity shall enforce stringent data protection measures for AI/ML data, both at rest and in motion, to safeguard its confidentiality and integrity." }, { "requirementID": "3.1.3 AI Supply Chain Security - 3", "requirementText": "The entity should document, develop, and maintain a record of all components used in AI/ML systems, including their sources, models, training data, libraries, versions, and any known vulnerabilities." }, { "requirementID": "3.2.1 Asset Management for AI/ML Systems - 1", "requirementText": "The entity should establish and maintain an accurate and comprehensive inventory of all AI/ML system components, including hardware, software, data sets, models, and algorithms (experimental and deployment)." }, { "requirementID": "3.2.1 Asset Management for AI/ML Systems - 2", "requirementText": "The asset inventory should be updated near real-time to account for changes such as additions, modifications, and retirements of assets." }, { "requirementID": "3.2.3 Patch Management - 5", "requirementText": "The entity should maintain a complete and up-to-date inventory of all AI/ML systems requiring patch management." }, { "requirementID": "3.2.5 Application Security - 4", "requirementText": "The entity should implement strong encryption and other appropriate security controls to protect data processed and generated by AI/ML applications." }, { "requirementID": "3.3.2 AI/ML Model Security - 1", "requirementText": "The entity should protect AI/ML models from unauthorized access and modification, by implementing measures like secure model storage, stringent access controls, and encryption both at rest and in motion." }, { "requirementID": "3.3.2 AI/ML Model Security - 2", "requirementText": "The entity should secure AI/ML models to an appropriate level during their entire lifecycle - from initial training and testing, through validation, to deployment and operational phases, employing strategies such as federated learning, or homomorphic encryption as required." }, { "requirementID": "3.3.4 Access to Training Data Control - 2", "requirementText": "Where possible, the entity should implement robust mechanisms to track and log all access to AI/ML training data, supporting auditability and accountability." }, { "requirementID": "3.3.4 Access to Training Data Control - 3", "requirementText": "The entity should apply data minimization practices to AI/ML training data, collecting and retaining only the necessary data for training purposes." }, { "requirementID": "3.3.4 Access to Training Data Control - 4", "requirementText": "The entity should ensure that AI/ML training data is protected against unauthorized modification, deletion, and exfiltration." }, { "requirementID": "3.3.4 Access to Training Data Control - 5", "requirementText": "Anonymization or pseudonymization should be applied when classified data is used for AI/ML training. Additionally, the entity should implement measures such as deleting datasets after training or encrypting data to protect training data." }, { "requirementID": "3.3.5 AI/ML Data Protection at Rest and in Motion - 1", "requirementText": "The entity should enforce robust encryption measures for AI/ML data both at rest and in motion to protect against unauthorized access and exfiltration based on the classification of the data." } ] } }, "Databricks": { "The Databricks AI Security Framework": { "link": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirements": [ { "requirementID": "DASF 6: Classify data", "requirementText": "Data catalogs can have a vast amount of data, often containing known and unknown sensitive data. It is critical for data teams to understand what kind of sensitive data exists in each table so that they can both govern and democratize access to this data.\nTo address this problem, Databricks Data Classification automatically classifies and tags tables in your catalog. This allows you to discover sensitive data, as well as apply governance controls over the results, using tools such as role-based access control (RBAC) and attribute-based access control (ABAC) policies in Unity Catalog. You can also use tags manually or via API as attributes containing keys and optional values that you can apply to different securable objects in Unity Catalog. Organizing securable objects with tags in Unity Catalog aids in efficient data management, data discovery and classification, essential for handling large datasets. Additionally, use governed tags to manage and enforce a tag policy. Tag policies allow admins to define which tag keys are governed, specify the allowed values for those tags, and control who can assign or modify them. This provides organizations with centralized control over tag usage, supporting consistent data classification, compliance, and operational standards." }, { "requirementID": "DASF 7: Enforce data quality checks on batch and streaming datasets\n", "requirementText": "Databricks Delta Live Tables (DLT) simplifies ETL development with declarative pipelines that integrate quality control checks and performance monitoring." }, { "requirementID": "DASF 10: Version data\n", "requirementText": "Store data in a lakehouse architecture using Delta tables. Delta tables can be versioned to revert any user or malicious actor poisoning of data. Data can be stored in a lakehouse architecture in the customer’s cloud account. Both raw data and feature tables are stored as Delta tables with access controls to determine who can read and modify them. Data lineage with UC helps track and audit changes and the origin of ML data sources. Each operation that modifies a Delta Lake table creates a new table version. User actions are tracked and audited, and lineage of transformations is available all in the same platform. You can use history information to audit operations, roll back a table or query a table at a specific point in time using time travel." }, { "requirementID": "DASF 11: Capture and view data lineage", "requirementText": "Unity Catalog tracks and visualizes real-time data lineage across all languages to the column level, providing a traceable history of an object from notebooks, workflows, models and dashboards. This enhances transparency and compliance, with accessibility provided through the Catalog Explorer." }, { "requirementID": "DASF 12: Delete records from datasets", "requirementText": "Data governance in Delta Lake, the lakehouse storage layer, utilizes its atomicity, consistency, isolation, durability (ACID) properties for effective data management. This includes the capability to remove data based on specific predicates from a Delta Table, including the complete removal of data’s history, supporting compliance with regulations like GDPR and CCPA." }, { "requirementID": "DASF 17: Track and reproduce the training data used for ML model training\n", "requirementText": "MLflow with Delta Lake tracks the training data used for ML model training. It also enables the identification of specific ML models and runs derived from particular datasets for regulatory and auditable attribution." }, { "requirementID": "DASF 18: Govern model assets\n", "requirementText": "With Unity Catalog, organizations can implement a unified governance framework for their structured and unstructured data, machine learning models, notebooks, features, functions, and files, enhancing security and compliance across clouds and platforms. Maintain an updated inventory of high-impact AI use cases, including details on purpose, benefits, risks, and risk management practices." }, { "requirementID": "DASF 22: Build models with all representative, accurate and relevant data sources\n", "requirementText": "Harnessing internal data and intellectual property to customize large AI models can offer a significant competitive edge. However, this process can be complex, involving coordination across various parts of the organization. The Data Intelligence Platform addresses this challenge by integrating data across traditionally isolated departments and systems. This integration facilitates a more cohesive data and AI strategy, enabling the effective training, testing and evaluation of models using a comprehensive dataset. Use caution when preparing data for traditional models and GenAI training to ensure that you are not unintentionally including data that causes legal conflicts, such as copyright violations, privacy violations or HIPAA violations." }, { "requirementID": "DASF 23: Register, version, approve, promote, deploy and monitor models\n", "requirementText": "MLflow Model Registry supports managing the machine learning model lifecycle with capabilities for lineage tracking, versioning, staging and model serving." }, { "requirementID": "DASF 28: Create model aliases, tags and annotations", "requirementText": "Model aliases in machine learning workflows allow you to assign a mutable, named reference to a specific version of a registered model. This functionality is beneficial for tracking and managing different stages of a model’s lifecycle, indicating the current deployment status of any given model version." }, { "requirementID": "DASF 29: Build MLOps workflows", "requirementText": "The lakehouse forms the foundation of a data-centric AI platform. Key to this is the ability to manage both data and AI assets from a unified governance solution on the lakehouse. Databricks Unity Catalog enables this by providing centralized access control, auditing, approvals, model workflow, lineage, and data discovery capabilities across Databricks workspaces.\nThese benefits are now extended to MLflow Models with the introduction of Models in Unity Catalog. Through providing a hosted version of the MLflow Model Registry in Unity Catalog, the full lifecycle of an ML model can be managed while leveraging Unity Catalog’s capability to share assets across Databricks workspaces and trace lineage across both data and models." }, { "requirementID": "DASF 58: Protect data with filters and masking\n", "requirementText": "Implement filters on sensitive table data using row filters and column masks by harnessing the power of Unity Catalog to secure your data at a granular level. Row filters allow you to apply a filter to a table so that queries return only rows that meet the filter criteria. Column masks let you apply a masking function to a table column." }, { "requirementID": "DASF 71: Log and register AI agents", "requirementText": "Logging an agent and registering it with catalog is the basis of the development and deployment process of agentica AI. The use of the Databricks AI Agent Framework, leveraging MLflow, to log and register all AI agents. Logging is the foundational step, capturing a specific \"point in time\" version of the agent's code, configuration, and Python environment to ensure traceability and evaluation quality.\nThe standard logging mechanism is MLflow Models from Code, which ensures the agent is packaged reliably for deployment. Key requirements during logging include:\n1. Model Signature: Defining the MLflow Model Signature to validate agent inputs and outputs, ensuring correct interaction with tools and other downstream applications.\n2. Resource Declaration: Explicitly declaring all external Databricks-managed resources (e.g., Vector Search indices, Foundation Model Serving Endpoints) that the agent requires for execution.\nFollowing successful logging, the agent must be registered to Unity Catalog (UC). This registration packages the agent as a model within UC, enabling the use of Unity Catalog permissions for authorization over the agent and its associated resources." }, { "requirementID": "DASF 72: Securely store and reuse agent state", "requirementText": "Managing Agent state is critical to secure multi-agent systems because it provides a centralized, protected layer for agents to store their short-term memory (such as conversation history, decision logs, or temporary tasks) directly within this governed perimeter, rather than exporting sensitive data to disconnected, insecure external databases." } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.2.1-1", "requirementText": "Developers, Data Custodians and System Operators shall maintain a comprehensive inventory of their assets (including their interdependencies/connectivity)." }, { "requirementID": "Provision 5.2.1-2", "requirementText": "As part of broader software security practices, Developers, Data Custodians and System Operators shall have processes and tools to track, authenticate, manage version control and secure their assets due to the increased complexities of AI specific assets." }, { "requirementID": "Provision 5.2.1-3", "requirementText": "System Operators shall develop and tailor their disaster recovery plans to account for specific attacks aimed at AI systems." }, { "requirementID": "Provision 5.2.1-3.1", "requirementText": "System Operators should ensure that a known good state can be restored." }, { "requirementID": "Provision 5.2.1-4", "requirementText": "Developers, System Operators, Data Custodians and End-users shall protect sensitive data, such as training or test data, against unauthorized access (see clause 5.2.3 for details on securing data)." }, { "requirementID": "Provision 5.2.1-4.1", "requirementText": "Developers, Data Custodians and System Operators shall apply checks and sanitisation to data and inputs when designing the model based on their access to said data and inputs and where those data and inputs are stored. This shall be repeated when model revisions are made in response to user feedback or continuous learning. See clause 5.2.2 for relevant provisions for open source." }, { "requirementID": "Provision 5.2.1-4.2", "requirementText": "Where training data or model weights could be confidential, Developers shall put proportionate protections in place." } ] }, "SAI 002 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_gr/SAI/001_099/002/01.01.01_60/gr_SAI002v010101p.pdf", "requirements": [ { "requirementID": "6.1.3 Supply chain security - 3", "requirementText": "Building data and model security considerations into the contracting processes." }, { "requirementID": "6.5 - Hash checks", "requirementText": "Existing cryptographic mechanisms can be used for protecting the integrity of data in an efficient way. For verification of data integrity there is a trade-off between efficiency and security, which should be balanced according to the risk level of the application." } ] }, "TR 104 048 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_tr/104000_104099/104048/01.01.01_60/tr_104048v010101p.pdf", "requirements": [ { "requirementID": "6.1.3 Supply chain security - 5", "requirementText": "Using additional security (for example cryptographic protection of data) to protect the most critical functions." }, { "requirementID": "6.5 Analysis - Fine-tuning", "requirementText": "Fine-tuning and/or regular retraining of models with locally-verified or otherwise trusted data, where possible." }, { "requirementID": "6.5 Analysis - Following standard cybersecurity supply chain guidance", "requirementText": "Following standard cybersecurity supply chain guidance. Data, models and the roles and risks associated with them can be understood and assessed in the same way as any other component of a system." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.2.1-1", "requirementText": "\"Developers, Data Custodians and System Operators shall maintain a comprehensive inventory of their assets (including their interdependencies / connectivity).\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout a clear understanding of AI assets and their dependencies, organizations can not be able to provide protection and risk exposing their AI systems to unauthorized access, data leakage, and vulnerability to external attacks.\n\nExample Measures/Controls:\nEstablish an AI Asset Inventory: Create and maintain a centralized inventory that records all AI assets, including datasets, models, software dependencies, hardware resources, and system configurations." }, { "requirementID": "Provision 5.2.1-2", "requirementText": "\"As part of broader software security practices, Developers, Data Custodians and System Operators shall have processes and tools to track, authenticate, manage version control, and secure their assets due to the increased complexities of AI specific assets.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout secure tracking, version control, and authentication, AI systems can be vulnerable to unauthorized changes, data integrity issues, and version conflicts, leading to compromised reliability and security. This is exacerbated by the rapid changes in Gen.AI tool and models.\n\nExample Measures/Controls 1:\nImplement AI Asset Tracking: Use version control and ML Ops systems to manage and track changes to AI assets including models, datasets, and related software components ensuring transparency and rollback capability.\n\nExample Measures/Controls 2:\nAuthenticate, Authorize and Log Access to Assets: Enforce strict access controls and authentication protocols to limit asset modifications to authorized personnel only logging any access." }, { "requirementID": "Provision 5.2.1-3", "requirementText": "\"System Operators shall develop and tailor their disaster recovery plans to account for specific attacks aimed at AI systems.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout tailored disaster recovery, AI systems can be unprepared for incidents such as data poisoning, or Large Language Model (LLM) weaponisation, leaving the organization vulnerable to prolonged disruption leading to data leakage, DoS or compromised model performance. Maintaining a reliable known good state can be challenging, particularly with continuous learning models or systems with frequent updates, increasing the risk of losing data integrity.\n\nExample Measures/Controls:\nIncorporate AI-Specific Threat Scenarios into Recovery Plans: Update disaster recovery plans to address AI-specific risks, including adversarial attacks, data poisoning, and model drift and ensure readiness for prompt recovery." }, { "requirementID": "Provision 5.2.1-3.1", "requirementText": "\"System Operators should ensure that a known good state can be restored.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nInability to restore a known good state can lead to prolonged system downtime, data loss, or operational disruptions after failures or attacks\n\nExample Measures/Controls 1:\nEstablish and Maintain a Known Good State: Regularly backup AI models, data, and system configurations, maintaining a known good state that can be restored after a disruption. Good state can contain additional internal state to help a model reach optimal performance after model warm-up. This might not be always possible, and a new model will have to be trained addressing the attack. Nevertheless, backups will help accelerate developing a mitigation.\n\nExample Measures/Controls 2:\nDevelop Recovery plans for advanced scenarios: Some incidents will include abuse and weaponisation of AI systems, especially Generative AI, to stage misinformation or other attacks abusing an organization's system. Recovering from such as attacks will require multi-disciplinary expertise and response strategies to minimize impact and harm." }, { "requirementID": "Provision 5.2.1-4", "requirementText": "\"Developers, System Operators, Data Custodians and End-users shall protect sensitive data, such as training or test data, against unauthorized access (see principle 7 for details on securing data).\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nUnauthorized access to sensitive data, such as training datasets, training algorithms, hyper parameters and model parameters can lead to privacy violations, data breaches, or compromised model integrity, increasing regulatory and reputational risks.\n\nExample Measures/Controls 1:\nImplement Data Encryption at Rest and in Transit: Encrypt sensitive data at rest and in transit to protect against unauthorized access, ensuring data confidentiality and security.\n\nExample Measures/Controls 2:\nImplement Strong Data Access Controls: Restrict access to sensitive data to authorized personnel only, using multi-factor authentication and role-based access controls. Programmatic updates via pipelines have strict RBAC and approvals in place to prevent abuse.\n\nExample Measures/Controls 3:\nData Access and Usage Monitoring: Implement automated monitoring tools to monitor access and usage of sensitive data including proprietary model weights, generating alerts for any unusual patterns or unauthorized attempts." }, { "requirementID": "Provision 5.2.1-4.1", "requirementText": "\"Developers, Data Custodians and System Operators shall apply checks and sanitisation to data and inputs when designing the model based on their access to said data and inputs and where those data and inputs are stored. This shall be repeated when model revisions are made in response to user feedback or continuous learning. See principle 6 for relevant provisions for open source.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nLack of data and input sanitization can introduce biases, errors, or malicious data, leading to compromised outputs, security risks, as well as potential legal and reputational harm.\n\nExample Measures/Controls:\nApply Data Sanitisation and Validation: Ensure that all data - both training datasets and runtime inputs - are sanitized and validated to prevent data poisoning, malicious inputs, and biases. During training, verify that datasets are clean, unbiased, and aligned with model objectives." }, { "requirementID": "Provision 5.2.1-4.2", "requirementText": "\"Where training data or model weights could be confidential, Developers shall put proportionate protections in place.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nFailure to adequately protect sensitive training data or model weights can lead to unauthorized access, intellectual property theft, or exposure of confidential information, increasing the risk of data breaches and regulatory non-compliance." } ] }, "TR 104 222 - Securing Artificial Intelligence; Mitigation Strategy Report": { "link": "https://www.etsi.org/deliver/etsi_tr/104200_104299/104222/01.02.01_60/tr_104222v010201p.pdf", "requirements": [ { "requirementID": "5.2.2 - Data sanitisation - 1", "requirementText": "Reject on negative impact (RONI), [i.11]): RONI was the first proposal and identifies outliers by training the model with and without each point and comparing the performance. Due to frequently retraining the addressed model, RONI's run-time overhead is significant, and its performance is also worse than later proposals." }, { "requirementID": "6.3.2", "requirementText": "Watermarking [i.79]: to protect the copyright of DNN models, the technique embeds watermarks in the training dataset that enable to verify the ownership of deployed DNN services. The method creates the watermarks in the training dataset, then assigns pre-defined labels to the different watermarks, and finally trains the watermarks with pre-defined labels to DNNs. The DNNs automatically learn and memorize the patterns of embedded watermarks and pre-defined labels. By doing this, the protected model will produce the pre-defined prediction upon watermarked queries, and so will models obtained by the first type of model stealing attack. The watermarking technique turns backdoor attacks into a mitigation. In order to detect this model stealing attack, the detection method only requires normal usage of the suspicious model with zero knowledge of model parameters, i.e. only requires access to its API. The technique is usually applied over image data." }, { "requirementID": "6.4.2 - 1", "requirementText": "Private Aggregation of Teacher Ensembles (PATE) [i.89] combines multiple models trained with disjoined datasets. These models are used as teachers for a student model, which learns to make prediction. The prediction is chosen by noisy voting among all of the teachers. These teacher models are not published and thus inaccessible except to the student model. PATE provides differential privacy for training data independent of the learning algorithm. Despite the provable robustness against membership inference attacks, PATE is hard to achieve with negligible utility loss." } ] }, "TS 104 224 - Securing Artificial Intelligence (SAI); Explicability and transparency of AI processing": { "link": "https://www.etsi.org/deliver/etsi_ts/104200_104299/104224/01.01.01_60/ts_104224v010101p.pdf", "requirements": [ { "requirementID": "7 (1)", "requirementText": "Data in transit, i.e. Connectivity, data transported through space, mathematical integrity shall be assured." }, { "requirementID": "7 (2)", "requirementText": "Data at rest, i.e. Storage, data transported through time, mathematical integrity shall be assured." }, { "requirementID": "7 (3)", "requirementText": "Data in process, i.e. Compute, data acted upon. In this case as the data is likely to be modified the integrity of the processing should be assured." } ] } }, "EU ": { "EU AI Act": { "link": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirements": [ { "requirementID": "15.4 Accuracy, Robustness and Cybersecurity", "requirementText": "High-risk AI systems shall be as resilient as possible regarding errors, faults or inconsistencies that may occur within the system or the environment in which the system operates, in particular due to their interaction with natural persons or other systems. Technical and organisational measures shall be taken in this regard.\n\nThe robustness of high-risk AI systems may be achieved through technical redundancy solutions, which may include backup or fail-safe plans.\n\nHigh-risk AI systems that continue to learn after being placed on the market or put into service shall be developed in such a way as to eliminate or reduce as far as possible the risk of possibly biased outputs influencing input for future operations (feedback loops), and as to ensure that any such feedback loops are duly addressed with appropriate mitigation measures." }, { "requirementID": "15.5 Accuracy, Robustness and Cybersecurity", "requirementText": "High-risk AI systems shall be resilient against attempts by unauthorised third parties to alter their use, outputs or performance by exploiting system vulnerabilities.\n\nThe technical solutions aiming to ensure the cybersecurity of high-risk AI systems shall be appropriate to the relevant circumstances and the risks.\n\nThe technical solutions to address AI specific vulnerabilities shall include, where appropriate, measures to prevent, detect, respond to, resolve and control for attacks trying to manipulate the training data set (data poisoning), or pre-trained components used in training (model poisoning), inputs designed to cause the AI model to make a mistake (adversarial examples or model evasion), confidentiality attacks or model flaws." }, { "requirementID": "26.4 Obligations of deployers of high-risk AI systems", "requirementText": "Without prejudice to paragraphs 1 and 2, to the extent the deployer exercises control over the input data, that deployer shall ensure that input data is relevant and sufficiently representative in view of the intended purpose of the high-risk AI system." }, { "requirementID": "26.9 Obligations of deployers of high-risk AI systems", "requirementText": "Where applicable, deployers of high-risk AI systems shall use the information provided under Article 13 of this Regulation to comply with their obligation to carry out a data protection impact assessment under Article 35 of Regulation (EU) 2016/679 or Article 27 of Directive (EU) 2016/680." } ] } }, "European Commission": { "Ethics guidelines for trustworthy AI": { "link": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirements": [ { "requirementID": "1.3.1 Privacy and data protection", "requirementText": "AI systems must guarantee privacy and data protection throughout a system’s entire lifecycle. This includes the information initially provided by the user, as well as the information generated about the user over the course of their interaction with the system it must be ensured that data collected about them will not be used to unlawfully or unfairly discriminate against them." } ] } }, "Federal Office for Information Security": { "AI Security Concerns in a Nutshell": { "link": "https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/KI/Practical_Al-Security_Guide_2023.pdf?__blob=publicationFile&v=5", "requirements": [ { "requirementID": "3.3 Defending against Evasion Attacks - Generalization", "requirementText": "Using a diverse and qualitative training data set is a good way to reduce the susceptibility of the model to certain adversarial examples. If the AI’s decision barriers enclose the known class too closely it may be easy to sample visually close inputs, which are detected as different class [6]. Additionally, random transformations within the bounds of the natural feature distribution, like omitting input pixels (dropout), tilting, compression, or filters can be used to increase the size and variety of the training data." }, { "requirementID": "4.5 Defending against Information Extraction Attacks - Decrease Model Output", "requirementText": "As many information extraction attacks use model confidence scores as the basis for an attack, reducing the scope of the model‘s output values or their precision might increase the effort for attackers [15]. However, as for example seen in the case of membership inference attacks, there might be attack methods just relying on class labels circumventing such a measure." }, { "requirementID": "4.5 Defending against Information Extraction Attacks - Data Sanitization", "requirementText": "Removing all the sensitive parts of the data before using it for training makes it impossible for intruders to extract the data from the trained model." }, { "requirementID": "4.5 Defending against Information Extraction Attacks - Avoid Overfitting", "requirementText": "Privacy attacks benefit from the overfitting of a model. Consequently, good model generalization mitigates the risk of successful privacy attacks. This might be achieved by a large and diverse training set as well as techniques such as regularization, dropout, or dataset condensation [16]. However, effectiveness of the used methods might depend on the concrete setting at hand." }, { "requirementID": "4.5 Defending against Information Extraction Attacks - Differential Privacy", "requirementText": "Differential privacy is a concept that helps to describe and quantify privacy in the processing of data. It demands, “Nothing about an individual should be learnable from the database that cannot be learned without access to the database” [17]. Differential privacy is often measured by a parameter ε, with lower values corresponding to greater privacy. Given a concrete application, the correct choice of ε is difficult to determine because there is a trade-off between privacy and the accuracy of the algorithm or model that uses the database. Finding suitable parameters might be costly in terms of computational effort. A model trained with differential private data might still be susceptible to attribute inference attacks since DP does not explicitly aim to protect attribute privacy [9]." } ] } }, "Google": { "Secure AI Framework": { "link": "https://www.saif.google/secure-ai-framework", "requirements": [ { "requirementID": "Training Data Management", "requirementText": "Ensure that all data used to train and evaluate models is authorized for the intended purposes." }, { "requirementID": "Training Data Sanitization", "requirementText": "Detect and remove or remediate poisoned or sensitive data in training and evaluation." }, { "requirementID": "Model and Data Inventory Management", "requirementText": "Ensure that all data, code, models, and transformation tools used in AI applications are inventoried and tracked." } ] } }, "IBM": { "IBM Framework for Securing Generative AI": { "link": "https://www.ibm.com/products/tutorials/ibm-framework-for-securing-generative-ai", "requirements": [ { "requirementID": "Secure the data", "requirementText": "During the data collection and handling phase, not only do you need to collect mounds and mounds of data to feed an AI model, but you’re also providing access to lots of different people, including data scientists, engineers, developers and others. There is an inherent risk presented in centralizing all that data in one place and granting various stakeholders—most of whom don’t have security experience—access to it.\n\nJust consider if intellectual property (IP) that is fundamental to the business is exposed due to mishandling of the training data, potentially creating an existential threat to the business. Leveraging troves of data for an AI model means organizations need to assess the varying risk tied to personally identifiable information (PII), privacy concerns and other sensitive information, and then place the proper security controls around that data.\n\nThe primary target in the data collection phase are the underlying data sets, with data exfiltration deemed as the likeliest technique that attackers will seek to employ to get their hands on valuable and monetizable information. As attackers seek the path of least resistance, underlying data sets are a blinking light promising a high yield.\n\nOrganizations must not overlook the importance of security fundamentals—in fact, they should be prioritized. If applied correctly, these fundamentals can have a substantial impact on an organization’s security posture. This includes focusing on data discovery and classification, encryption at rest and in transit, and key management delivered from data security platforms such as IBM Security® Guardium®. This also means focusing on identity and access management fundamentals enforced by solutions such as IBM Security® Verify, which help ensure that no single entity has unrestricted access to the AI model. Finally, organizations must raise security awareness with the data scientists and researchers and make sure security teams work closely with those teams to ensure proper guardrails." } ] } }, "ICO": { "Guidance on the AI Auditing Framework - Draft guidance for consultation ": { "link": "https://ico.org.uk/media2/about-the-ico/consultations/2617219/guidance-on-the-ai-auditing-framework-draft-for-consultation.pdf", "requirements": [ { "requirementID": "What steps should we take to manage the risks of privacy attacks on AI models? - 1", "requirementText": "If you train models and provide them to others, you should assess whether those models may contain personal data or are at risk of revealing it if attacked and take appropriate steps to mitigate these risks.\nYou should assess whether the training data contains identified or identifiable personal data of individuals, either directly or by those who may have access to the model. You should assess the means that may be reasonably likely to be used, in light of the vulnerabilities described above. As this is a rapidly developing area, you should stay up-to-date with the state of the art in both methods of attack and mitigation." }, { "requirementID": "Preventative Controls - 5", "requirementText": "Have an approach for asset management to ensure a coordinated approach to the optimisation of costs, risks, service/performance and sustainability." } ] } }, "ISO": { "42001:2023 - Information technology — Artificial intelligence — Management system": { "link": "https://www.iso.org/standard/42001", "requirements": [ { "requirementID": "9.3.2", "requirementText": "Management review inputs" } ] } }, "ISO/IEC": { "DIS 27090": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:27090:dis:ed-1:v1:en", "requirements": [ { "requirementID": "7.7", "requirementText": "Malicious input detection and filtering" } ] }, "DIS 5181": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:5181:dis:ed-1:v1:en", "requirements": [ { "requirementID": "5.2", "requirementText": "Data provenance methodology" }, { "requirementID": "6.2", "requirementText": "Data flow control model" }, { "requirementID": "6.3", "requirementText": "Data provenance reference architecture model" }, { "requirementID": "6.4", "requirementText": "Data provenance security and privacy" } ] } }, "METI (Japan)": { "Governance Guidelines for Implementation of AI Principles": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20220128_2.pdf", "requirements": [ { "requirementID": "Action Target 3-1-3", "requirementText": "Companies that provide data should provide information on the data sets including data collection sources, collection policies, collection criteria, annotation criteria, and limitation on use to ensure that companies that develop AI systems are able to appropriately conduct gap analysis, and AI system developers should acquire data sets from data providers that provide sufficient information." } ] } }, "MIC/METI (Japan)": { "AI Guidelines for Business": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirements": [ { "requirementID": "Safety - 3 (a)", "requirementText": "In accordance with the characteristics and purposes of AI systems and services, ensure the accuracy, and recency as necessary, of the data (appropriateness of the data) to be used for training." } ] } }, "Microsoft": { "Cloud Adoption Framework - Secure AI": { "link": "https://learn.microsoft.com/en-us/azure/cloud-adoption-framework/scenarios/ai/secure", "requirements": [ { "requirementID": "Secure AI resources\n1 - Create a complete AI asset inventory", "requirementText": "Unknown AI assets create security unguarded sides that attackers exploit to gain unauthorized access. A comprehensive inventory enables effective monitoring and rapid incident response for all AI components. Use Azure Resource Graph Explorer to discover AI resources across subscriptions, implement Microsoft Defender for Cloud to identify generative AI workloads, and maintain this inventory through automated scanning and regular validation." }, { "requirementID": "Secure AI data\n2 - Implement comprehensive data loss prevention", "requirementText": "Unauthorized data exposure through AI responses can compromise sensitive information and violate regulatory requirements. Data loss prevention controls prevent AI models from inadvertently revealing protected data in their outputs. Use Microsoft Purview Data Loss Prevention to scan and block sensitive data in AI workflows. Configure content filtering to prevent sensitive information leakage, and implement custom filters to detect and redact organization-specific sensitive data patterns. For Microsoft Copilot Studio, Configure data loss prevention policies for agents." }, { "requirementID": "Secure AI data\n3 - Protect AI artifacts from compromise", "requirementText": "Unsecured AI models and datasets become targets for theft, poisoning, or reverse engineering attacks. Protected artifacts maintain intellectual property value and prevent malicious manipulation of AI systems. Store models and datasets in Azure Blob Storage with private endpoints, apply encryption at rest and in transit, and implement strict access policies with monitoring to detect unauthorized access attempts." } ] }, "Responsible AI Standard": { "link": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirements": [ { "requirementID": "A4.2", "requirementText": "Define and document procedures for the collection and processing of data, to include annotation, labelling, cleaning, enrichment, and aggregation, where relevant." }, { "requirementID": "RS2.3", "requirementText": "Document your plan for managing previously unknown failures that come to light once the system is in use:\n1) Describe the system’s rollback plan and document the time that may elapse until the entire system, across all endpoints can be rolled back.\n2) Describe support for turning features off and document the time that may elapse until the feature can be turned off across all endpoints.\n3) Describe the process for updating and releasing updates to each model and document the time that may elapse until the system has been updated across all endpoints.\n4) Describe how customers, partners, and end users will be notified of changes to the system, updated understandings of failures, and their best mitigations." } ] } }, "MITRE": { "ATLAS Framework": { "link": "https://atlas.mitre.org/mitigations", "requirements": [ { "requirementID": "AML.M0003 - Model Hardening", "requirementText": "Use techniques to make AI models robust to adversarial inputs such as adversarial training or network distillation." }, { "requirementID": "AML.M0010 - Input Restoration", "requirementText": "Preprocess all inference data to nullify or reverse potential adversarial perturbations." }, { "requirementID": "AML.M0011 - Restrict Library Loading", "requirementText": "Prevent abuse of library loading mechanisms in the operating system and software to load untrusted code by configuring appropriate library loading mechanisms and investigating potential vulnerable software.\n\nFile formats such as pickle files that are commonly used to store AI models can contain exploits that allow for loading of malicious libraries." }, { "requirementID": "AML.M0012 - Encrypt Sensitive Information", "requirementText": "Encrypt sensitive data such as AI models to protect against adversaries attempting to access sensitive data." }, { "requirementID": "AML.M0013 - Code Signing", "requirementText": "Enforce binary and application integrity with digital signature verification to prevent untrusted code from executing. Adversaries can embed malicious code in AI software or models. Enforcement of code signing can prevent the compromise of the AI supply chain and prevent execution of malicious code." }, { "requirementID": "AML.M0014 - Verify AI Artifacts", "requirementText": "Verify the cryptographic checksum of all AI artifacts to verify that the file was not modified by an attacker." }, { "requirementID": "AML.M0033 - Input and Output Validation for AI Agent Components", "requirementText": "Implement validation on inputs and outputs for the tools and data sources used by AI agents. Validation includes enforcing a common data format, schema validation, checks for sensitive or prohibited information leakage, and data sanitization to remove potential injections or unsafe code. Input and output validation can help prevent compromises from spreading in AI-enabled systems and can help secure the workflow when multiple components are chained together. Validation should be performed external to the AI agent." } ] }, "SAFE-AI": { "link": "https://atlas.mitre.org/pdf-files/SAFEAI_Full_Report.pdf", "requirements": [ { "requirementID": "Model poisoning", "requirementText": "AI-enabled systems may be vulnerable to attacks that perturb AI model inputs, or modify AI models to undermine their reliability, integrity, and availability. A wide variety of model poisoning attacks are possible - such as making changes to the code, objective functions, model parameters, or training data - so the attack surface is potentially very large. Mitigations include controlling access to models and data, continual/continuous testing, and establishing baselines for data distributions and model performance." }, { "requirementID": "Data poisoning", "requirementText": "Poisoned data can compromise the decision making of an AI-enabled system and bias its outputs. An adversary can poison data by compromising external data datasets, or by gaining access to the system and poisoning data stored for training, testing, or other operations. This is an important concern for AI because data poisoning attacks can embed vulnerabilities into an AI-enabled system that may be difficult to detect. For example, an adversary may embed a backdoor trigger that gets activated by designated input data during operations and generates the adversary’s desired output rather than the correct response. Mitigations include preprocessing all data to sanitize and validate it before it is used, and continual/continuous testing." }, { "requirementID": "Model exposure", "requirementText": "Attackers may try to gain knowledge about the models in an AI-enabled system to steal intellectual property, enable unauthorized use of model capabilities, or achieve some competitive advantage. Attackers may exploit a variety of attack vectors to gain access to models, such as coding errors and software vulnerabilities in the system, weak access controls, and poor protection management practices for AI assets, to extract a trained AI model directly, or collect enough information about the model architecture to create a functionally equivalent copy of the model. Consequently, it is often prudent to treat models in AI-enabled systems as sensitive assets requiring protection and controlled access. Mitigations include stringent access controls (especially to prevent data exfiltration), and strong asset protection management practices." }, { "requirementID": "Sensitive data exposure", "requirementText": "Sensitive data must be safeguarded during the development, testing, and deployment of an AI-enabled system. When attackers obtain unauthorized access to sensitive data, the resulting privacy breach and data exposure can compromise the confidentiality of the data, in addition to facilitating data poisoning attacks that may be more effective when informed by privileged information. Countermeasures include implementing strong access controls, using secure and encrypted data storage, regularly updating and patching the AI models, and using data anonymization techniques." }, { "requirementID": "Sensitive information disclosure", "requirementText": "There are many ways AI applications can inadvertently disclose sensitive information, proprietary algorithms, or confidential data. For example, sensitive data may not be adequately filtered from AI responses, AI might memorize sensitive details during training, or there may be unintended data leaks due to misinterpretation of a query. In addition, adversaries may craft prompts that induce the AI to leak sensitive information from proprietary training data, data sources the AI component is connected to, or information from other users of the AI component. Disclosures like this can lead to unauthorized access, intellectual property theft, and privacy breaches. To mitigate these risks, AI applications should employ data sanitization, implement appropriate usage policies, and restrict the types of data returned by the AI component." }, { "requirementID": "Loss of data", "requirementText": "AI-enabled systems have a strong dependence and reliance on data during all phases of the lifecycle. Malicious destruction or corruption of data is therefore a critical AI concern. All potential vulnerabilities an attacker can exploit to gain access to a system and its data need to be anticipated, including outdated or unpatched software components, weak or improperly implemented access controls, and poor asset protection management practices. Key considerations for avoiding data loss include access controls in general ( and write access in particular), along with careful assessment of backup and recovery capacity to avoid any backup capability limitations." }, { "requirementID": "Unauthorized access to data", "requirementText": "Unauthorized access to data is a concern that is important across all aspects of an AI-enabled system (including AI platforms, tools, and models) and all phases of the system lifecycle. The associated security risks include data breaches, manipulation of AI models, exposure of sensitive information, and potential misuse of AI systems for malicious purposes. Preventive measures include implementing strong access controls, using encrypted data storage, regularly updating and patching the AI models, and using AI-powered security tools for threat detection and response. Regular audits and security assessments can also help identify potential vulnerabilities. It may also be useful to recognize an expanded set of roles for the purposes of access control in AI-enabled systems (e.g. roles for prompt templating and model generation)." }, { "requirementID": "Backup capability limitations", "requirementText": "AI-enabled systems have a strong dependence and reliance on data during all phases of the lifecycle. Backup capability limitations that may weaken safeguards against data loss or data corruption are therefore an important AI concern. Several issues make it challenging to provide suitable backup capabilities for an AI-enabled system: the data volumes associated with AI systems are massive and far beyond those for other software systems; complex AIenabled computations may produce data usage patterns that change drastically during routine operations; and data storage requirements may differ in the different phases of the AI lifecycle. Mitigations include careful assessment of backup and recovery capacity requirements for each stage of the system lifecycle, along with specific backup policies to address the key data usage patterns." }, { "requirementID": "Publicly-available product or service", "requirementText": "Adversaries may research existing open source or other publicly-available implementations of machine learning attacks. Adversaries may target AI-enabled products or services to gain access to the underlying AI model. Adversaries may use the product or service to indirectly access the AI model, potentially revealing details about the model, its algorithms, or its inferences through logs or metadata. This type of access can expose sensitive information about the model's structure, parameters, or decision-making processes, and business intelligence. By exploiting these vulnerabilities, adversaries can gain insights that may help them craft more effective adversarial attacks or reverse-engineer the model. The research community often publishes their code for reproducibility and to further future research. Libraries intended for research purposes, such as CleverHans, the Adversarial Robustness Toolbox, and FoolBox, can be weaponized by an adversary." }, { "requirementID": "Metadata exposure", "requirementText": "The threat of metadata exposure involves the temporary logging of metadata about API requests, such as the time received, frequency and size of the request, and the IP addresses from which the requests originated. While this logging aims to improve the service and combat abuse, it could inadvertently reveal patterns or usage information. Adversaries could analyze the metadata to infer sensitive details about document processing activities, operational behaviors, and timelines. Additionally, IP addresses could be exploited to track user locations or launch targeted attacks against specific networks. Although the document content itself is not directly exposed, the metadata could still provide valuable insights to the adversaries attempting to cause harm" } ] } }, "Multi Agency": { "Guidelines for secure AI system development": { "link": "https://www.ncsc.gov.uk/files/Guidelines-for-secure-AI-system-development.pdf", "requirements": [ { "requirementID": "Identify, track and protect your assets", "requirementText": "You understand the value to your organisation of your AI-related assets, including models, data (including user feedback), prompts, software, documentation, logs and assessments (including information about potentially unsafe capabilities and failure modes), recognising where they represent significant investment and where access to them enables an attacker. You treat logs as sensitive data and implement controls to protect their confidentiality, integrity and availability.\n\nYou know where your assets reside and have assessed and accepted any associated risks. You have processes and tools to track, authenticate, version control and secure your assets, and can restore to a known good state in the event of compromise.\n\nYou have processes and controls in place to manage what data AI systems can access, and to manage content generated by AI according to its sensitivity (and the sensitivity of the inputs that went into generating it)." } ] } }, "NCSC/NSA/CISA etc": { "AI Data Security\n": { "link": "https://media.defense.gov/2025/May/22/2003720601/-1/-1/0/CSI_AI_DATA_SECURITY.PDF", "requirements": [ { "requirementID": "3.2 Data Sanitization", "requirementText": "Sanitize the training data by applying techniques like data filtering, sampling, and normalization. This helps reduce the impact of outliers, noisy data, and other potentially poisoned inputs, ensuring that models learn from highquality, representative datasets. Perform sanitization on a regular basis, especially prior to each and every training, fine-tuning, or any other process that adjusts model parameters." }, { "requirementID": "3.3 Secure Training Pipelines", "requirementText": "Secure data collection, pre-processing, and training pipelines to prevent malicious actors from tampering with datasets or model parameters." }, { "requirementID": "3.5 Data anonymization", "requirementText": "Implement anonymization techniques to protect sensitive data attributes, keeping them confidential while allowing AI models to learn patterns and generate accurate predictions." }, { "requirementID": "3.7 Metadata Validation", "requirementText": "Establish data validation processes to check the completeness and consistency of metadata before data is used for AI training." }, { "requirementID": "3.8 Data Encrichment", "requirementText": "Use available resources, such as reference data and trusted third-party data, to supplement missing metadata and improve the overall quality of the training data." }, { "requirementID": "3.9 Regular Training Data Audits ", "requirementText": "Regularly audit training data to detect, assess, and address potential issues that can result in systematically inaccurate AI systems." }, { "requirementID": "3.13 Remove Inaccurate Information from Training Data", "requirementText": "Identify and remove inaccurate or misleading information from AI datasets to the extent feasible." }, { "requirementID": "3.14 Data Provenance and Verification", "requirementText": "Implement provenance verification mechanisms during data collection to help ensure that only accurate and reliable data is used. This process can include methods such as cross-verification, fact-checking, source analysis, data provenance tracking, and content credentials." }, { "requirementID": "3.15 Add More Training Data", "requirementText": "Increasing the amount of non-malicious data makes training more robust against poisoned examples—provided that these poisoned examples are small in number. One way to do this is through data augmentation— the creation of artificial training set samples that are small variations of existing samples. The goal is to “outnumber” the poisoned samples so the model “forgets” them. Note that this mitigation can only be applied during training, and therefore does not apply to an already trained model. [28]" }, { "requirementID": "3.16 Data Quality Control", "requirementText": "Perform quality control on data including detecting poisoned samples through integrity checks, statistical deviation, or pattern recognition. Proactively implement data quality controls during the training phase to prevent issues before they arise in production." }, { "requirementID": "3.17 Data Deduplication", "requirementText": "Implement deduplication techniques (such as fuzzy matching, hashing, clustering, etc.) to carefully identify and handle duplicates and nearduplicates in the data." }, { "requirementID": "4.2 Data-Quality Testing ", "requirementText": "AI system developers should use data-quality assessment tools to assist in selecting and filtering data used for model training or adaptation. Understanding the current dataset and its impact on model behavior is critical to detecting data drift." } ] } }, "NIST": { "AI 800-1": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirements": [ { "requirementID": "Practice 3.2: Maintain security practices sufficient to prevent unauthorized access - 2", "requirementText": "Apply security practices tailored to the context of foundation models, such as protections against exfiltrating large amounts of data (e.g., model weights) and other vulnerabilities (e.g., extraction attacks)." } ] }, "AI RMF 1.0": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirements": [ { "requirementID": "GOVERN 1.6", "requirementText": "Mechanisms are in place to inventory AI systems and are resourced according to organizational risk priorities." }, { "requirementID": "MANAGE 2.1", "requirementText": "Resources required to manage AI risks are taken into account – along with viable non-AI alternative systems, approaches, or methods – to reduce the magnitude or likelihood of potential impacts." } ] }, "IR 8596: Cybersecurity Framework Profile for Artificial Intelligence (Cyber AI Profile): NIST Community Profile": { "link": "https://csrc.nist.gov/pubs/ir/8596/iprd", "requirements": [ { "requirementID": "ID.AM-01", "requirementText": "Inventories of hardware managed by the organization are maintained" }, { "requirementID": "ID.AM-02", "requirementText": "Inventories of software, services, and systems managed by the organization are maintained" }, { "requirementID": "ID.AM-03", "requirementText": "Representations of the organization’s authorized network communication and internal and external network data flows are maintained" }, { "requirementID": "ID.AM-05", "requirementText": "Assets are prioritized based on classification, criticality, resources, and impact on the mission" }, { "requirementID": "ID.AM-07", "requirementText": "Inventories of data and corresponding metadata for designated data types are maintained" }, { "requirementID": "ID.RA-01", "requirementText": "Vulnerabilities in assets are identified, validated, and recorded" }, { "requirementID": "PR.DS-01", "requirementText": "The confidentiality, integrity, and availability of data-at-rest are protected" }, { "requirementID": "PR.DS-02", "requirementText": "The confidentiality, integrity, and availability of data-intransit are protected" }, { "requirementID": "PR.DS-03", "requirementText": "The confidentiality, integrity, and availability of data-inuse are protected" }, { "requirementID": "RC.RP-05", "requirementText": "The integrity of restored assets is verified, systems and services are restored, and normal operating status is confirmed" } ] }, "SP 800-218A": { "link": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirements": [ { "requirementID": "PO.3.2", "requirementText": "Follow recommended security practices to deploy, operate, and maintain tools and toolchains.\n\nExecute the plan to develop and implement automated toolchains that secure AI model development and reduce human effort, especially at the scale often used by AI models.\n\nVerify the security of toolchains at a frequency commensurate with risk." }, { "requirementID": "PO.4.2", "requirementText": "Implement processes, mechanisms, etc. to gather and safeguard the necessary information in support of the criteria." }, { "requirementID": "PS.1.1", "requirementText": "Store all forms of code – including source code, executable code, and configuration-as-code – based on the principle of least privilege so that only authorized personnel, tools, services, etc. have access.\n\nSecure code storage should include AI models, model weights, pipelines, reward models, and any other AI model elements that need their confidentiality, integrity, and/or availability protected. These elements do not all have to be stored in the same place or through the same type of mechanism.\n\nFollow the principle of least privilege to minimize direct access to AI models and model elements regardless of where they are stored or executed.\n\nStore reward models separately from AI models and data.\n\nPermit indirect access only to model weights." }, { "requirementID": "PS.1.2", "requirementText": "Protect all training, testing, finetuning, and aligning data from unauthorized access and modification.\n\nContinuously monitor the confidentiality (for non-public data only) and integrity of training, testing, fine-tuning, and aligning data." }, { "requirementID": "PS.1.3", "requirementText": "Protect all model weights and configuration parameter data from unauthorized access and modification.\n\nKeep model weights and configuration parameters separate from training, testing, fine-tuning, and aligning data.\n\nContinuously monitor the confidentiality (for closed models only) and integrity of model weights and configuration parameters.\n\nFollow the principle of least privilege to restrict access to AI model weights, configuration parameters, and services during development.\n\nSpecify and implement additional riskproportionate cybersecurity practices around model weights, such as encryption, cryptographic hashes, digital signatures, multiparty authorization, and air-gapped environments." }, { "requirementID": "PW.3.1", "requirementText": "Analyze data for signs of data poisoning, bias, homogeneity, and tampering before using it for AI model training, testing, fine-tuning, or aligning purposes, and mitigate the risks as necessary.\n\nVerify the provenance (when known) and integrity of training, testing, fine-tuning, and aligning data before use.\n\nSelect and apply appropriate methods for analyzing and altering the training, testing, finetuning, and aligning data for an AI model. Examples of methods include anomaly detection, bias detection, data cleaning, data curation, data filtering, data sanitization, factchecking, and noise reduction." }, { "requirementID": "PW.4.1", "requirementText": "Acquire and maintain wellsecured software components (e.g., software libraries, modules, middleware, frameworks) from commercial, opensource, and other third-party developers for use by the organization’s software." }, { "requirementID": "PW.6.2", "requirementText": "Determine which compiler, interpreter, and build tool features should be used and how each should be configured, then implement and use the approved configurations." } ] } }, "OWASP": { "AI Exchange": { "link": "https://owaspai.org/docs/ai_security_overview/", "requirements": [ { "requirementID": "1.2 General controls for sensitive data limitation - DATA MINIMIZE", "requirementText": "Data minimize: remove data fields or records (e.g. from a training set) that are unnecessary for the application, in order to prevent potential data leaks or manipulation because we cannot leak what isn’t there in the first place." }, { "requirementID": "1.2 General controls for sensitive data limitation - ALLOWED DATA", "requirementText": "Ensure allowed data, meaning: removing data (e.g. from a training set) that is prohibited for the intended purpose. This is particularly important if consent was not given and the data contains personal information collected for a different purpose." }, { "requirementID": "1.2 General controls for sensitive data limitation - SHORT RETAIN", "requirementText": "Short retain: Remove or anonymize data once it is no longer needed, or when legally required (e.g., due to privacy laws)." }, { "requirementID": "1.2 General controls for sensitive data limitation - OBFUSCATE TRAINING DATA", "requirementText": "Obfuscate training data: attain a degree of obfuscation of sensitive data where possible." } ] }, "LLM Top 10": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirements": [ { "requirementID": "LLM01: Prompt Injection - 6", "requirementText": "Separate and clearly denote untrusted content to limit its influence on user prompts." }, { "requirementID": "LLM02: Sensitive Information Disclosure - 1", "requirementText": "Implement data sanitization to prevent user data from entering the training model. This includes scrubbing or masking sensitive content before it is used in training.\n" }, { "requirementID": "LLM02: Sensitive Information Disclosure - 2", "requirementText": "Apply strict input validation methods to detect and filter out potentially harmful or sensitive data inputs, ensuring they do not compromise the model." }, { "requirementID": "LLM02: Sensitive Information Disclosure - 4", "requirementText": "Limit model access to external data sources, and ensure runtime data orchestration is securely managed to avoid unintended data leakage." }, { "requirementID": "LLM02: Sensitive Information Disclosure - 6", "requirementText": "Apply techniques that add noise to the data or outputs, making it difficult for attackers to reverse-engineer individual data points.\n" }, { "requirementID": "LLM02: Sensitive Information Disclosure - 11", "requirementText": "Use homomorphic encryption to enable secure data analysis and privacy-preserving machine learning. This ensures data remains confidential while being processed by the model." }, { "requirementID": "LLM02: Sensitive Information Disclosure - 12", "requirementText": "Implement tokenization to preprocess and sanitize sensitive information. Techniques like pattern matching can detect and redact confidential content before processing.\n" }, { "requirementID": "LLM03: Supply Chain - 1", "requirementText": "Carefully vet data sources and suppliers, including T&Cs and their privacy policies, only using trusted suppliers. Regularly review and audit supplier Security and Access, ensuring no changes in their security posture or T&Cs." }, { "requirementID": "LLM04: Data and Model Poisoning - 6", "requirementText": "Use data version control (DVC) to track changes in datasets and detect manipulation. Versioning is crucial for maintaining model integrity." }, { "requirementID": "LLM05: Improper Data Handling - 1", "requirementText": "Treat the model as any other user, adopting a zero-trust approach, and apply proper input validation on responses coming from the model to backend functions." }, { "requirementID": "LLM05: Improper Data Handling - 2", "requirementText": "Follow the OWASP ASVS (Application Security Verification Standard) guidelines to ensure effective input validation and sanitization." }, { "requirementID": "LLM06: Excessive Agency - 8", "requirementText": "Follow secure coding best practice, such as applying OWASP’s recommendations in ASVS (Application Security Verification Standard), with a particularly strong focus on input sanitisation. Use Static Application Security Testing (SAST) and Dynamic and Interactive application testing (DAST, IAST) in development pipelines.\n" }, { "requirementID": "LLM08: Vector and Embedding Weaknesses - 2", "requirementText": "Implement robust data validation pipelines for knowledge sources. Regularly audit and validate the integrity of the knowledge base for hidden codes and data poisoning. Accept data only from trusted and verified sources." }, { "requirementID": "LLM10: Unbounded Consumption - 1", "requirementText": "Implement strict input validation to ensure that inputs do not exceed reasonable size limits." }, { "requirementID": "LLM10: Unbounded Consumption - 14", "requirementText": "Use a centralized ML model inventory or registry for models used in production, ensuring proper governance and access control." } ] }, "OWASP Model Context Protocol (MCP) Top 10": { "link": "https://owasp.org/www-project-mcp-top-10/", "requirements": [ { "requirementID": "MCP01:2025 - Token Mismanagement and Secret Exposure - 1", "requirementText": "Implement Secret Hygiene Controls\n- Store secrets in secure vaults (e.g., HashiCorp Vault, AWS Secrets Manager).\n- Use environment variable injection only at runtime, never at build time." }, { "requirementID": "MCP01:2025 - Token Mismanagement and Secret Exposure - 5", "requirementText": "Enforce Governance Controls\n- Define organizational policies for credential lifecycle management.\n- Regularly audit MCP configurations, server endpoints, and stored contexts.\n- Use Hardware Security Modules (HSMs) or Secrets Managers (AWS Secrets Manager, HashiCorp Vault, etc.) for runtime injection." }, { "requirementID": "MCP03:2025 - Tool Poisoning - 1", "requirementText": "Signed Schemas & Manifest Integrity\n- Digitally sign schemas and tool manifests (e.g., JWS / COSE / PKI-backed signatures). Agents must verify signatures before accepting or using a schema.\n- Use content-addressable identifiers (hashes) for schema versions and validate against trusted hashes." }, { "requirementID": "MCP03:2025 - Tool Poisoning - 2", "requirementText": "Immutable Schema Registry & Version Control\n- Store schemas in an immutable version-controlled system (Git with signed commits) or an append-only ledger.\n- Enforce branch protections, required code review, and multi-person approval for schema changes." }, { "requirementID": "MCP03:2025 - Tool Poisoning - 5", "requirementText": "Schema Provenance & Metadata\n- Each schema/version should include provenance metadata: author, signature, hash, timestamp, and approved-by.\n- Agents should log the schema hash and provenance metadata used for each invocation for audit and forensic purposes." }, { "requirementID": "MCP03:2025 - Tool Poisoning - 8", "requirementText": "Roll back agents to the last known-good schema hash and force revalidation." }, { "requirementID": "MCP04:2025 – Software Supply Chain Attacks & Dependency Tampering - 1", "requirementText": "Signed Components & Provenance Verification Require cryptographic signing for:\n- SDKs\n- Plugins\n- Tool manifests\n- Container images\n- Validate signatures during install + startup" }, { "requirementID": "MCP04:2025 – Software Supply Chain Attacks & Dependency Tampering - 3", "requirementText": "Track:\n- Versions\n- Hashes\n- Licenses\n- Provenance metadata" }, { "requirementID": "MCP06:2025 – Intent Flow Subversion - 3", "requirementText": "Unified Context Sanitization & Validation (Untrusted-by-Default)\n- Treat all natural-language content from MCP resources/ or tool outputs as untrusted.\n- Apply the same prompt-injection safeguards defined in OWASP LLM01:2025 to all retrieved context before it can influence agent planning or behavior." }, { "requirementID": "MCP06:2025 – Intent Flow Subversion - 4", "requirementText": "Strict Context Tagging & Metadata Sandboxing\n- Leverage MCP metadata to tag retrieved content as [UNTRUSTED_CONTEXT]. Instruct the model to treat content within these tags as passive data, never as executable instructions or policy overrides." }, { "requirementID": "MCP10:2025 – Context Injection & Over-Sharing - 3", "requirementText": "Data Classification Tagging\nTag all inputs and retrieved data as:\nPublic\nInternal\nConfidential\nRestricted - Prevent low-trust or cross-domain agents from accessing restricted context." }, { "requirementID": "MCP10:2025 – Context Injection & Over-Sharing - 5", "requirementText": "Context Sanitization & Redaction\nScan and redact:\nPII\nSecrets\nTokens Internal system identifiers before storing in context. - Use automated scanners or classification pipelines." } ] }, "OWASP Top 10 for Agentic Applications for 2026": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "requirements": [ { "requirementID": "ASI01: Agent Goal Hijack - 1", "requirementText": "Treat all natural-language inputs (e.g., user-provided text, uploaded documents, retrieved content) as untrusted. Route them through the same input-validation and prompt-injection safeguards defined in LLM01:2025 before they can influence goal selection, planning, or tool calls." }, { "requirementID": "ASI01: Agent Goal Hijack - 6", "requirementText": "Sanitize and validate any connected data source - including RAG inputs, emails, calendar invites, uploaded files, external APIs, browsing output, and peer-agent messages - using CDR, prompt- carrier detection, and content filtering before the data can influence agent goals or actions." }, { "requirementID": "ASI05: Unexpected Code Execution (RCE) - 1", "requirementText": "Follow the mitigations of LLM05:2025 Improper Output Handling with input validation and output encoding to sanitize agent-generated code." }, { "requirementID": "ASI10: Rogue Agents - 5", "requirementText": "Identity Attestation and Behavioral Integrity Enforcement: Implement per-agent cryptographic identity attestation and enforce behavioral integrity baselines throughout the agent lifecycle. Attach signed behavioral manifests declaring expected capabilities, tools, and goals that are validated by orchestration services before each action. Integrate a behavioral verification layer that continuously monitors tasks for deviations from the declared manifest for e.g. unapproved tool invocations, unexpected data exfiltration attempts etc." }, { "requirementID": "ASI10: Rogue Agents - 7", "requirementText": "Recovery and Reintegration: Establish trusted baselines for restoring quarantined or remediated agents. Require fresh attestation, dependency verification, and human approval before reintegration into production networks." } ] } }, "Personal Data Protection Commission Singapore (PDPC)": { "Model Artificial Intelligence Governance Framework Second Edition": { "link": "https://www.pdpc.gov.sg/-/media/files/pdpc/pdf-files/resource-for-organisation/ai/sgmodelaigovframework2.pdf", "requirements": [ { "requirementID": "Reproducibility - d)", "requirementText": "For companies that procure commercial off-the-shelf AI systems, checking with the original AI solution provider about whether the model’s results are reproducible;" } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirements": [ { "requirementID": "12.6", "requirementText": "An Entity must put in place proper mechanisms to ensure the confidentiality and security of information that the outsourcing service provider may have access to." }, { "requirementID": "15.8", "requirementText": "An Entity should adopt an effective data governance framework specifically designed for Al to ensure that data used by the Al Model is accurate, complete, consistent, secure, and provided in a timely manner for the Al System to function as designed. The framework should document the extent to which the data meets an Entity's requirements for data quality, gaps in data quality that may exist and steps an Entity will take, where possible, to resolve these gaps over time." }, { "requirementID": "15.9", "requirementText": "An Entity that develops high risk systems should use techniques involving the training of models with data developed on the basis of training, validation and Testing Data Sets when:\nAn Entity's training, validation, and Testing Data Sets must be subject to appropriate design choices, data governance and management practices. Those practices must concern, the following:\n- Data collection processes.\n- Relevant data preparation processing operations, such as annotation, labelling, cleaning, enrichment, and aggregation.\n- The formulation of relevant assumptions related to the Entity's ability to acquire suitable data sets to allow the development of systems, notably with respect to the information that the data are supposed to measure and represent.\n- A prior assessment of the availability, quantity and suitability of the Data Sets that are needed.\n- Examination in view of possible Biases that are likely to affect health and safety of natural persons or lead to discrimination.\n- The identification of any possible data gaps or shortcomings, and how those gaps and shortcomings can be addressed." }, { "requirementID": "15.11", "requirementText": "An Entity should ensure the Data Sets are as free of errors and complete as possible in view of the intended purpose of the Al System and sourced from reputable vendors." }, { "requirementID": "17.8", "requirementText": "An Entity that utilizes Al must deploy a Data Loss Prevention (DLP) tool to protect against sensitive data loss." }, { "requirementID": "19.3.3", "requirementText": "Maintain records of the various versions of the model including its code should establish a robust system for versioning and then maintaining a record of each version of the Al Model." }, { "requirementID": "19.3.4", "requirementText": "Archive original Data Sets used to develop, re-train or calibrate models." } ] } }, "SANS": { "Critical AI Security Guidelines": { "link": "https://sansorg.egnyte.com/dl/bvkYQxrW8QMj", "requirements": [ { "requirementID": "1.2 Protecting Augmentation Data", "requirementText": "In retrieval-augmented generation (RAG) architectures, vectorDBs are commonly used to store and retrieve semantically indexed data that is fed into LLMs. However, augmentation data used in these systems can be a source of significant risk if not properly secured. Protecting augmentation data requires more than just applying access controls. Data stored in these databases should be treated as sensitive, especially if it influences LLM responses. If tampered with, this data can cause models to generate misleading or dangerous outputs. In addition to enforcing least privilege access models for both read and write operations, organizations should implement secure upload pipelines, logging and auditing of changes, and validation mechanisms to detect unauthorized modifications. Encryption at rest and in transit, along with digital signing of documents or chunks, can enhance trust in the augmentation layer." }, { "requirementID": "2.1 Defend Training Data", "requirementText": "1. Models are only as good as their training data. Adversarial access can negatively impact training data, hiding malicious activities. \n2. Not just in the context of LLMs; be concerned with data used to train a non-LLM model that will make security or operational decisions." }, { "requirementID": "2.2 Avoid Data Commingling", "requirementText": "1. Leveraging enterprise data allows for better grounded applications. \n2. Sensitive data should be sanitized or anonymized prior to LLM incorporation." }, { "requirementID": "2.3 Limit Sensitive Prompt Content ", "requirementText": "1. Attackers with unauthorized access to an organization’s prompts can infer sensitive information such as internal business processes, proprietary data, decision logic, or even personally identifiable information (PII). \n2. Avoid including sensitive or confidential information in prompts whenever possible, to reduce the risk of data exposure." }, { "requirementID": "4.1 Establish LLM Guardrails", "requirementText": "Guardrails are rules that instruct a model on how to respond or avoid responding to specific topics. These guardrails can be set in various ways. They can be created manually by searching for explicit values in the prompt or response, or they can be built in by the LLM hosting provider." }, { "requirementID": "4.2 Sanitize, Validate, and Filter LLM Inputs/Prompts", "requirementText": "Prompt injection represents the most common LLM application attack vector and warrants a multilayered approach to protection and detection. All prompts should be preprocessed prior to inference and all model outputs postprocessed prior to response. If employing RAG, additional LLM input filtering and validation would need to occur after the prompt has been augmented." }, { "requirementID": "4.7 Encoding/Decoding", "requirementText": "Many foundation models, even relatively small ones, often can handle input and output using different encoding schemes. Encoded prompt/response data might be able to bypass security, safety, and alignment measures. Testing by this paper’s authors has shown models often could handle Base64, Hex, or Morse encoded data input without even being explicitly told the formatting or asking for decoding" }, { "requirementID": "4.8 Compression/Decompression", "requirementText": "Another means of input/output obfuscation available to adversaries could include methods of compression and decompression. Support for handling various compression and decompression schemes varies substantially across model implementations." }, { "requirementID": "6.4 Model Registries", "requirementText": "Model registries are centralized repositories that track and manage ML models through their life cycle, from development to deployment. These can be a valuable addition to your AI deployment workflows, providing security and governance benefits. Registries track model versions, dependencies, and training data, ensuring full traceability and enabling rollback, if needed." }, { "requirementID": "7.4 Establish a model integrity baseline", "requirementText": "Use cryptographic hashes, model registries, and periodic validation checks to confirm deployed models have not been altered" } ] } }, "SDAIA (Saudi Arabia)": { "AI Ethics Principles": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Documents/ai-principles.pdf", "requirements": [ { "requirementID": "Principle 2 – Privacy & Security - Plan and Design - 6", "requirementText": "Security mechanisms for de-identification should be planned for the sensitive or personal data in the system. Furthermore, read/write/update actions should be authorized for the relevant groups." }, { "requirementID": "Principle 2 – Privacy & Security - Prepare Input Data - 1", "requirementText": "The exercise of data procurement, management, and organization should uphold the legal frameworks and standards of data privacy. Data privacy and security protect information from a wide range of threats." }, { "requirementID": "Principle 2 – Privacy & Security - Prepare Input Data - 3", "requirementText": "Designers and engineers of the AI system must exhibit the appropriate levels of integrity to safeguard the accuracy and completeness of information and processing methods to ensure that the privacy and security legal framework and standards are followed. They should also ensure that the availability and storage of data are protected through suitable security database systems." }, { "requirementID": "Principle 2 – Privacy & Security - Prepare Input Data - 4", "requirementText": "All processed data should be classified to ensure that it receives the appropriate level of protection in accordance with its sensitivity or security classification and that AI system developers and owners are aware of the classification or sensitivity of the information they are handling and the associated requirements to keep it secure. All data shall be classified in terms of business requirements, criticality, and sensitivity in order to prevent unauthorized disclosure or modification. Data classification should be conducted in a contextual manner that does not result in the inference of personal information. Furthermore, de-identification mechanisms should be employed based on data classification as well as requirements relating to data protection laws." }, { "requirementID": "Principle 2 – Privacy & Security - Prepare Input Data - 5", "requirementText": "Data backups and archiving actions should be taken in this stage to align with business continuity, disaster recovery and risk mitigation policies." }, { "requirementID": "Principle 2 – Privacy & Security - Deploy and Monitor - 2", "requirementText": "AI System Owners should be accountable for the design and implementation of AI systems in such a way as to ensure that personal information is protected throughout the life cycle of the AI system. The components of the AI system should be updated based on continuous monitoring and privacy impact assessment." }, { "requirementID": "Principle 5 – Reliability & Safety - Prepare Input Data - 1", "requirementText": "Adequate steps and actions should be taken to measure the data sample’s quality, accuracy, suitability, and credibility when dealing with the data sets of an AI model. This is essential to ensure the accuracy of data interpretation by the AI system, the consistency of avoiding misleading measurements, as well as ensuring the relevance of the AI system’s outcomes to the purpose of the model." }, { "requirementID": "Principle 7 – Accountability & Responsibility - Prepare Input Data - 1", "requirementText": "An important aspect of the Accountability and Responsibility principle during Prepare Input Data step in the AI System Lifecycle is data quality as it affects the outcome of the AI model and decisions accordingly. It is, therefore, important to do necessary data quality checks, clean data and ensure the integrity of the data in order to get accurate results and capture intended behavior in supervised and unsupervised models." }, { "requirementID": "Principle 7 – Accountability & Responsibility - Prepare Input Data - 2", "requirementText": "Data sets should be approved and signed-off before commencing with developing the AI model. Furthermore, the data should be cleansed from societal biases. In parallel with the fairness principle, the sensitive features should not be included in the model data. In the event that sensitive features need to be included, the rationale or trade-off behind the decision for such inclusion should be clearly explained. The data preparation process and data quality checks should be documented and validated by responsible parties." } ] }, "Generative AI Guidelines": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Files/GenerativeAIPublicEN.pdf", "requirements": [ { "requirementID": "4.5 Privacy & Security - 1", "requirementText": "Ensure adequate privacy and security measures in place when using classified data to train GenAI modules. " }, { "requirementID": "4.5 Privacy & Security - 2", "requirementText": "Implement rigorous data protection measures and consider the principles such as, the one outlined in the Personal Data Protection Law." } ] } }, "Smart Dubai (UAE)": { "AI Ethics Principles & Guidelines": { "link": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirements": [ { "requirementID": "1.2.2.4", "requirementText": "In designing AI systems to inform significant decisions, AI developer organisations should consider measures to maintain data accuracy over time, including:\n• the completeness of the data;\n• timely update of the data, and;\n• whether the context in which the data was collected a_x001D_ects its suitability for the intended use case" } ] } }, "TAIBOM": { "Bringing Trustworthiness to AI-Enabled Systems": { "link": "https://arxiv.org/pdf/2510.02169", "requirements": [ { "requirementID": "UC 1: Declaring Training Data for Transparency", "requirementText": "AI model training frequently involves datasets compiled from disparate or opaque sources. Without structured, verifiable records of data composition, organisations face difficulties validating dataset provenance, understanding licensing implications, or reproducing experimental outcomes." }, { "requirementID": "UC 2: Assessing Training Data for Poisoning", "requirementText": "Data poisoning introduces malicious or manipulated samples into training datasets, often without detection. Detecting poisoning requires dataset versioning, reproducibility, and traceability of data inputs to model outputs." }, { "requirementID": "UC 3: Detecting Training and Inference System Tampering", "requirementText": "Post-training tampering of AI components, such as code, configurations, or model weights, can result in erroneous or malicious behaviour. Mitigation requires strong binding between training inputs and deployed inference artifacts." } ] } }, "U.S. Department of Health & Human Services": { "Trustworthy AI (TAI) Playbook: Executive Summary": { "link": "https://www.hhs.gov/sites/default/files/hhs-trustworthy-ai-playbook-executive-summary.pdf", "requirements": [ { "requirementID": "Privacy", "requirementText": "Individual, group, or entity privacy should be respected, and their data should not be used beyond its intended and stated use; data used has been approved by the data owner or steward" } ] } }, "UAE Ministry of Cabinet Affairs": { "The UAE Charter for the Development and Use of Artificial Intelligence": { "link": "https://uaelegislation.gov.ae/en/policy/details/the-uae-charter-for-the-development-and-use-of-artificial-intelligence#:~:text=The%20charter%20covers%20the%20following%20priorities%20and,and%20use%20of%20AI%20in%20the%20country.", "requirements": [ { "requirementID": "4. Data Privacy", "requirementText": "In line with the UAE’s stance on privacy rights, while data is essential for AI development, supporting and promoting innovation in AI, the privacy of community members remains a top priority." } ] } } }, "Principle 6": { "CEN/CENELEC": { "prEN 40000-1-2: Cybersecurity requirements for products with digital elements - Part 1-2: Principles for cyber resilience": { "link": "https://genorma.com/en/standards/pren-40000-1-2", "requirements": [ { "requirementID": "7.4", "requirementText": "Cybersecurity architecture and design" } ] } }, "Central Bank of the UAE": { "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E": { "link": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirements": [ { "requirementID": "6. Continuous Monitoring and Review - c", "requirementText": "Mechanisms should be in place to detect, report and remediate any performance issues, biases or unintended consequences that may arise over time." } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirements": [ { "requirementID": "1.2.3 Secure Deployment ", "requirementText": "Deploy the AI system using methods that maintain its security posture, including using proper network segmentation and access control, as well as verifying and validating that the AI system works as intended." }, { "requirementID": "2.2.5 - Data Silos", "requirementText": "Address the complexity of integrating AI systems in OT environments due to OT/IT network segregation, proprietary protocols, formats, diversity of OT products, and a multitude of suppliers." }, { "requirementID": "2.4.1 ", "requirementText": "When integrating AI into OT environments, critical infrastructure owners and operators should carefully evaluate the existing infrastructure to ensure compatibility and security." }, { "requirementID": "4.1.7 - Prefer push-based or brokered architectures that move required features or summaries out of OT without\ngranting persistent inbound access.", "requirementText": "Where data must traverse to business networks, use one-way transfer patterns and audited staging buffers. This method for implementing segmentation helps operators maintain existing segmentation best practices, such that the AI system is not a persistent attack path into OT." } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "AIS-06", "requirementText": "Establish and implement strategies and capabilities for secure, standardized, and compliant application deployment. Automate where possible." }, { "requirementID": "AIS-10", "requirementText": "Define and implement processes, procedures, and technical measures to secure APIs. Review and update for any improvements at least annually or after significant system changes." }, { "requirementID": "AIS-13", "requirementText": "Implement sandboxing techniques to execute AI tools and plugins in isolated environments to prevent unintended interactions with critical systems or data and limit the possibility of lateral movement." }, { "requirementID": "BCR-03", "requirementText": "Establish strategies to reduce the impact of business disruptions, and improve resiliency and recovery from business disruptions." }, { "requirementID": "BCR-04", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain a business continuity plan based on the results of the operational resilience strategies and capabilities." }, { "requirementID": "BCR-08", "requirementText": "Periodically perform backups. Ensure the confidentiality, integrity and availability of the backup, and verify restoration from backup for resiliency." }, { "requirementID": "BCR-09", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain a disaster response plan to recover from natural and man-made disasters. Update the plan at least annually or upon significant changes." }, { "requirementID": "BCR-10", "requirementText": "Exercise the disaster response plan annually or upon significant changes, including, if possible, participation of local emergency authorities." }, { "requirementID": "BCR-11", "requirementText": "Supplement business-critical equipment with both locally redundant and geographically dispersed equipment located at a reasonable minimum distance in accordance with applicable industry standards." }, { "requirementID": "CCC-08", "requirementText": "Implement a procedure for the management of exceptions, including emergencies, in the change and configuration process. Align the procedure with the requirements of GRC-04: Policy Exception Process." }, { "requirementID": "CCC-09", "requirementText": "Define and implement a process to proactively roll back changes to a previous known good state in case of errors or security concerns." }, { "requirementID": "CEK-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for Cryptography, Encryption and Key Management. Review and update the policies and procedures at least annually or upon significant changes." }, { "requirementID": "CEK-05", "requirementText": "Establish a standard change management procedure, to accommodate changes from internal and external sources, for review, approval, implementation and communication of cryptographic, encryption and key management technology changes." }, { "requirementID": "CEK-10", "requirementText": "Generate Cryptographic keys using industry accepted cryptographic libraries specifying the algorithm strength and the random number generator used." }, { "requirementID": "CEK-12", "requirementText": "Rotate cryptographic keys in accordance with the calculated cryptoperiod, which includes provisions for considering the risk of information disclosure and legal and regulatory requirements." }, { "requirementID": "CEK-13", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to revoke and remove cryptographic keys prior to the end of its established cryptoperiod, when a key is compromised, or an entity is no longer part of the organization, which include provisions for legal and regulatory requirements." }, { "requirementID": "CEK-14", "requirementText": "Define, implement, and evaluate processes, procedures, and technical measures to securely destroy cryptographic keys when they are no longer needed, which include provisions for legal and regulatory requirements." }, { "requirementID": "CEK-15", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to create keys in a pre-activated state when they have been generated but not authorized for use, which include provisions for legal and regulatory requirements." }, { "requirementID": "CEK-17", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to deactivate keys at the time of their expiration date, which include provisions for legal and regulatory requirements." }, { "requirementID": "CEK-19", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to use compromised keys to encrypt information only in controlled circumstance, and thereafter exclusively for decrypting data and never for encrypting data, which include provisions for legal and regulatory requirements." }, { "requirementID": "DCS-07", "requirementText": "Design and implement physical security perimeters to safeguard personnel, data, and information systems." }, { "requirementID": "DCS-09", "requirementText": "Allow only authorized personnel access to secure areas, with all ingress and egress points restricted, documented, and monitored by physical access control mechanisms. Retain access control records on a periodic basis as deemed appropriate by the organization." }, { "requirementID": "DCS-11", "requirementText": "Train datacenter personnel to safely manage adverse events, including but not limited to unauthorized ingress and egress attempts." }, { "requirementID": "DSP-23", "requirementText": "Regularly validate the consistency and conformity of training, fine-tuning or augmentation data. Implement dataset versioning to ensure traceability and enforce restrictions to prevent unauthorized changes." }, { "requirementID": "IAM-01", "requirementText": "Establish, document, approve, communicate, implement, apply, evaluate and maintain policies and procedures for identity and access management. Review and update the policies and procedures at least annually, or upon significant changes." }, { "requirementID": "IAM-03", "requirementText": "Manage, store, and regularly review the inventory of identities, and monitor their level of access." }, { "requirementID": "IAM-04", "requirementText": "Employ the separation of duties principle when implementing information system access." }, { "requirementID": "IAM-05", "requirementText": "Employ the least privilege principle when implementing information system access." }, { "requirementID": "IAM-07", "requirementText": "De-provision or modify identity access in a timely manner." }, { "requirementID": "IAM-09", "requirementText": "Define, implement and evaluate processes, procedures and technical measures for the segregation of privileged access roles." }, { "requirementID": "IAM-10", "requirementText": "Define and implement an access process to ensure privileged access roles and rights are granted for a time limited period, and implement procedures to prevent the accumulation of segregated privileged access." }, { "requirementID": "IAM-16", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to verify access to data and system functions is authorized." }, { "requirementID": "IPY-03", "requirementText": "Implement cryptographically secure and standardized network protocols for the management, import and export of data, according to industry standards." }, { "requirementID": "I&S-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for infrastructure and virtualization security. Review and update the policies and procedures at least annually, or upon significant changes." }, { "requirementID": "I&S-03", "requirementText": "Monitor, encrypt and restrict communications between environments to only authenticated and authorized connections, as justified by the business. Review these configurations at least annually, and support them by a documented justification of all allowed services, protocols, ports, and compensating controls." }, { "requirementID": "I&S-04", "requirementText": "Harden host and guest OS, hypervisor or infrastructure control plane, according to their respective best practices, and supported by technical controls, as part of a security baseline." }, { "requirementID": "I&S-05", "requirementText": "Separate production and non-production environments." }, { "requirementID": "I&S-07", "requirementText": "Use secure and encrypted communication channels when migrating servers, services, applications, or data to hosted environments. Such channels must include only up-to-date and approved protocols." }, { "requirementID": "I&S-09", "requirementText": "Define, implement and evaluate processes, procedures and defense-in-depth techniques for protection, detection, and timely response to network-based attacks." }, { "requirementID": "SEF-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for Security Incident Management, E-Discovery, and Forensics. Review and update the policies and procedures at least annually or upon significant changes." }, { "requirementID": "SEF-02", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for the timely management of security incidents. Review and update the policies and procedures at least annually, or upon significant changes." }, { "requirementID": "SEF-09", "requirementText": "Define incident categories and severity levels for AI systems, and determine response procedures for each, including automated response where applicable." }, { "requirementID": "TVM-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures to identify, report and prioritize the remediation of vulnerabilities and threats, in order to protect systems against vulnerability exploitation. Review and update the policies and procedures at least annually or upon significant changes." }, { "requirementID": "TVM-03", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to enable both scheduled and emergency responses to vulnerability identifications, based on the identified risk." }, { "requirementID": "TVM-07", "requirementText": "Define, implement and evaluate processes, procedures and technical measures based on identified risks to support scheduled and emergency responses to vulnerability identification." }, { "requirementID": "TVM-09", "requirementText": "Define and implement a process for tracking and reporting vulnerability identification and remediation activities that includes stakeholder notification." }, { "requirementID": "UEM-05", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to enforce policies and controls for all endpoints permitted to access systems and/or store, transmit, or process organizational data." }, { "requirementID": "UEM-08", "requirementText": "Protect information from unauthorized disclosure on managed endpoint devices with storage encryption." }, { "requirementID": "UEM-09", "requirementText": "Configure managed endpoints with anti-malware detection and prevention technology and services." }, { "requirementID": "UEM-10", "requirementText": "Configure managed endpoints with properly configured software firewalls." } ] } }, "CoSAI": { "AI Incident Response Framework": { "link": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirements": [ { "requirementID": "3.3.2. Detection and Analysis Phase - Detection Mechanisms - Integration Points", "requirementText": "• SIEM integration\n• Correlation with network security\n• Authentication event linking" }, { "requirementID": "3.3.3. Containment, Eradication, and Recovery Phase - Containment Strategies - Short-term Containment", "requirementText": "• Emergency prompt filters\n• Rate limiting/access restrictions\n• Component isolation\n• User/IP blocking\n" }, { "requirementID": "3.3.3. Containment, Eradication, and Recovery Phase - Eradication Procedures - Component Remediation", "requirementText": "• Data-Level: Remove poisoned data, reconstruct embeddings\n• Model-Level: Roll back to secure versions, enhance guardrails\n• Deployment-Level: Update prompts, strengthen controls\n• Output-Level: Implement filtering, content moderation" } ] }, "Establish Risks and Controls for the AI Supply Chain": { "link": "https://github.com/cosai-oasis/ws1-supply-chain/blob/main/risks-and-controls-for-the-ai-supply-chain-v1.md", "requirements": [ { "requirementID": "3.1.2 Model Training - Model Source Tampering", "requirementText": "Unauthorized modification of model architecture code to introduce backdoors:\nTwo-person code review requirements, secure source repositories with access controls" }, { "requirementID": "3.1.2 Model Training - Model Weight Tampering", "requirementText": "Unauthorized modification of model weights after training, during checkpoints, or in pre-trained models:\nAccess controls for model storage, cryptographic model signatures" }, { "requirementID": "3.1.3 Application Integration in the AI Supply Chain - Vulnerable Communication Channels", "requirementText": "Interception or manipulation of model-application traffic:\nEnd-to-end encryption, certificate validation, response signing" }, { "requirementID": "3.1.3 Application Integration in the AI Supply Chain - Third-Party API Compromise", "requirementText": "Security failures in external services integrated into the application:\nAPI security assessments, traffic monitoring, fallback mechanisms" }, { "requirementID": "3.2.3 Application - Indirect Prompt Injection", "requirementText": "Unauthorized instructions or malicious commands introduced to the model through external data sources during inference operations, including RAG systems, agentic frameworks, web search functionality, tool outputs, and other integrated data providers:\nImplement comprehensive access control frameworks with strong authentication mechanisms for all external data sources. Deploy advanced content filtering systems with specific detection capabilities for injection patterns across all integrated data channels." }, { "requirementID": "3.2.4 Infrastructure - Cache Poisoning", "requirementText": "Malicious manipulation of cached data or results within the AI infrastructure, where attackers introduce corrupted information into temporary storage mechanisms used to optimize model performance:\nImplement comprehensive cache validation protocols, cryptographic verification of cached content, and regular cache refresh cycles with integrity checks." }, { "requirementID": "3.2.4 Infrastructure - Overly Permissive Entitlements", "requirementText": "Excessive access rights within the AI infrastructure allowing components or users to access, modify, or control resources beyond operational requirements, creating unnecessary attack surfaces:\nImplement comprehensive least-privilege architecture, regularly audit entitlement configurations, enforce just-in-time access protocols, and establish proper service account isolation." } ] }, "Model Context Protocol (MCP) Security": { "link": "https://github.com/cosai-oasis/ws4-secure-design-agentic-systems/blob/main/model-context-protocol-security.md", "requirements": [ { "requirementID": "3.2.1 Agent Identity", "requirementText": "All requests should be traceable across the entire execution chain: the end user or initiating agent, any intermediate MCP servers, and the tools or services that performed the resulting actions. Standards are emerging to define the identity of agents and servers. One of these is SPIFFE / SPIRE, which provides cryptographic workload identities that can be granted authorization to resources. The SPIFFE ID can be used in token exchange as the subject or actor depending on the flow.\n\nSecure identity, authentication, and authorization across the agentic and MCP ecosystem is an extremely active area of research and development. We will provide a much deeper analysis of the problem space in a subsequent white paper." }, { "requirementID": "3.2.4 Cryptographic Integrity and Remote Attestation", "requirementText": "Hardware Trusted Execution Environments (TEE) like Intel TDX, and AMD-SEV/SNP provide stronger isolation and can provide protection against runtime tampering of trusted servers, such as tool poisoning due to server compromise. Combined with remote attestation, TEEs can isolate MCP servers and clients from compromised hardware, malicious administrators of server infrastructure, and certain classes of co-tenancy threats. For containerized deployments, consider the use of confidential containers (CoCo) that run containers in TEEs, and use remote attestation to verify the trustworthiness of the TEEs and what is running in them." }, { "requirementID": "3.2.5 Sandboxing and Isolation", "requirementText": "Agents and MCP servers should be executed with least privilege. MCP servers that interact with the host environment (e.g. by accessing files, running commands, issuing network connections), or that execute LLM-generated code, should always run in a sandbox to mitigate against potential safety and security threats.\n\nLLM-generated code and commands may contain hallucinations, bugs, or vulnerabilities, and should not run with full user privileges. MCP servers are commonly deployed in containers for ease of use, but containers should not be relied upon as a strong security boundary. Consider additional sandboxing (gVisor, Kata Containers, SELinux sandboxes) for stronger isolation." }, { "requirementID": "3.2.7 Transport Layer Security", "requirementText": "MCP is structured around distinct communication layers that facilitate robust interaction between systems. At the transport layer, MCP leverages two primary communication methods:\n\nstdio Transport:\nA direct, pipe-based stream communication channel, typically used for intra-process or tightly integrated inter-process communications. JSON-RPC messages flow directly via standard input/output streams. This transport is most commonly used for local servers.\n\nHTTP Streaming Transport:\nA generalized HTTP-based transport channel supporting bidirectional JSON-RPC communication via streamed request-response patterns. This transport is most commonly used for remote servers.\n\nAt the higher-level protocol layer, MCP employs JSON-RPC 2.0 to standardize the formatting and processing of commands and responses communicated across these transport channels. JSON-RPC ensures structured messaging, enabling interoperability and clarity of communication across diverse platforms.\n\nHowever, these transport and protocol layers, when improperly secured or configured, can expose MCP clients and servers to multiple vulnerabilities." }, { "requirementID": "3.2.11 Lifecycle and Governance a)", "requirementText": "Organizations must:\n- implement mandatory code signing verification for all MCP servers before installation,\n- use private package repositories with security scanning and approval workflows,\n- deploy software composition analysis (SCA) tools to detect vulnerable dependencies,\n- implement allow-lists of approved MCP servers with documented security reviews,\n- Run MCP servers and clients in TEEs and use remote attestation to verify prior to interactions,\n- use cryptographic hash verification for package integrity,\n- and deploy binary authorization that prevents execution of unsigned or unverified code." }, { "requirementID": "3.2.11 Lifecycle and Governance c)", "requirementText": "Lifecycle management demands:\n- maintaining centralized inventory of all deployed MCP servers with metadata (version, owner, purpose),\n- implementing automated discovery to detect shadow or unauthorized MCP deployments,\n- deploying lifecycle policies that automatically deprecate or remove outdated servers,\n- using configuration management tools (Ansible, Puppet, Chef) to maintain consistent deployments,\n- implementing rollback capabilities for problematic updates,\n- and deploying update management processes with testing and staged rollout." } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "2.2.6", "requirementText": "The entity shall apply robust network security controls to AI/ML systems to ensure confidentiality, integrity, and availability, tailored to the systems' specific needs and characteristics." }, { "requirementID": "2.3.4", "requirementText": "The entity shall enforce robust access controls on AI/ML training data to ensure data confidentiality, integrity, and privacy, while supporting accountability and auditability." }, { "requirementID": "2.4.2", "requirementText": "The entity shall ensure the operational resilience of AI/ML systems by incorporating redundancy and failover mechanisms to maintain availability during disruptions." }, { "requirementID": "2.4.3", "requirementText": "The entity shall proactively address AI/ML system through business continuity and disaster recovery planning processes and regularly testing restoration capability and continuity scenarios." }, { "requirementID": "2.6.2", "requirementText": "The entity shall establish a robust and efficient incident reporting and management system for AI/ML cyber security incidents that ensures prompt response, compliance with legal and contractual obligations, and supports continuous learning and improvement." }, { "requirementID": "3.1.4 Change Management and Reporting - 4", "requirementText": "Where possible, the entity should have a rollback plan in place to restore the system to its previous state in case a change introduces significant issues or risks." }, { "requirementID": "3.2.2 Security Configuration Management - 1", "requirementText": "The entity should establish and enforce secure configuration baselines for all AI/ML supporting infrastructure and applications." }, { "requirementID": "3.2.2 Security Configuration Management - 3", "requirementText": "The entity should establish procedures for the secure backup and recovery of AI/ML system configurations to minimize downtime and data loss in the event of a security incident or system failure." }, { "requirementID": "3.2.4 Vulnerability Management for AI/ML Systems - 1", "requirementText": "The entity should implement a robust vulnerability management program to identify, assess, and remediate security vulnerabilities in AI/ML systems." }, { "requirementID": "3.2.4 Vulnerability Management for AI/ML Systems - 4", "requirementText": "The entity should regularly review and update the vulnerability management program to align with changes in AI/ML technologies, threat landscape, and regulatory changes." }, { "requirementID": "3.2.5 Application Security - 3", "requirementText": "The entity should implement a robust access control mechanism in AI/ML applications to limit the access to AI/ML data and functions based on the principle of least privilege." }, { "requirementID": "3.2.6 Network Security for AI/ML Infrastructure - 1", "requirementText": "The entity should establish and enforce a network security policy specifically tailored for AI/ML infrastructure." }, { "requirementID": "3.2.6 Network Security for AI/ML Infrastructure - 2", "requirementText": "The entity should segment high risk AI/ML systems from the rest of the network to minimize the risk of lateral movement in case of a breach." }, { "requirementID": "3.2.6 Network Security for AI/ML Infrastructure - 3", "requirementText": "Network traffic to and from AI/ML systems should be securely encrypted and continuously monitored for any signs of intrusion or abnormal behavior." }, { "requirementID": "3.2.6 Network Security for AI/ML Infrastructure - 4", "requirementText": "The entity should implement firewall rules and Intrusion Detection/Prevention Systems (IDS/IPS), or other security technologies specifically configured for the unique network behaviors and requirements of AI/ML systems." }, { "requirementID": "3.2.6 Network Security for AI/ML Infrastructure - 5", "requirementText": "The entity should ensure that any remote access to AI/ML infrastructure is secure and controlled, utilizing secure protocols and methods such as multi-factor authentication." }, { "requirementID": "3.3.4 Access to Training Data Control - 1", "requirementText": "The entity should enforce strict access control measures for AI/ML training data, ensuring that only authorized individuals and processes can access this data and only it for its intended purpose." }, { "requirementID": "3.3.5 AI/ML Data Protection at Rest and in Motion - 2", "requirementText": "The entity should implement secure methods for the transfer of AI/ML data, using only approved and secure protocols based on the classification of the data." }, { "requirementID": "3.3.5 AI/ML Data Protection at Rest and in Motion - 3", "requirementText": "The entity should enforce strong key management practices for encryption keys used to protect AI/ML data." }, { "requirementID": "3.4.2 Fail-Safe & Backup for AI/ML Systems - 2", "requirementText": "The entity should design AI/ML systems, based on their criticality, with failover mechanisms to ensure system availability during disruptions, such as redundant infrastructure deployment in geographically diverse data centers, load balancing, and automatic failover and switching." }, { "requirementID": "3.4.2 Fail-Safe & Backup for AI/ML Systems - 3", "requirementText": "The entity should employ backup measures for AI/ML systems as appropriate, including periodic and secure backup of training data, model parameters, and system configurations." }, { "requirementID": "3.4.2 Fail-Safe & Backup for AI/ML Systems - 4", "requirementText": "The entity should regularly test and validate the effectiveness of fail-safe and backup measures to ensure their readiness in the event of a system failure or other disruptions." }, { "requirementID": "3.4.3 Continuity Planning for AI/ML Systems - 1", "requirementText": "The entity should consider AI/ML systems in business continuity planning and disaster recovery strategies." }, { "requirementID": "3.4.3 Continuity Planning for AI/ML Systems - 2", "requirementText": "The entity should establish measures to ensure the continuity of AI/ML systems, considering potential disruptions from both technical and non-technical factors, including data corruption, loss of critical resources, and sustained outages." }, { "requirementID": "3.4.3 Continuity Planning for AI/ML Systems - 3", "requirementText": "In cases where AI/ML systems are critical to operations, redundant systems or alternative processes should be available and ready for immediate deployment." }, { "requirementID": "3.4.3 Continuity Planning for AI/ML Systems - 4", "requirementText": "The entity should have a defined process to recover AI/ML systems from disruptions and outline key personnel, processes, resources, and critical dependencies necessary for the recovery of AI/ML systems." }, { "requirementID": "3.4.3 Continuity Planning for AI/ML Systems - 5", "requirementText": "The entity should periodically test the continuity and recovery plan for AIML systems and train relevant staff to ensure they are adequately prepared." }, { "requirementID": "3.5.2 Defending Against AI/ML Attacks - 5", "requirementText": "AI/ML systems should be designed and configured to resist, contain, and recover from attacks, minimizing potential damage and disruption." }, { "requirementID": "3.6.2 Incident Reporting and Management for AI/ML - 1", "requirementText": "The entity should establish a robust incident reporting and management process for AI/ML security incidents to ensure they are appropriately identified, logged, investigated, and resolved." }, { "requirementID": "3.6.2 Incident Reporting and Management for AI/ML - 4", "requirementText": "The entity should consider automated response mechanisms for AI/ML security incidents to mitigate the impact of attacks and prevent further escalation such as isolating affected systems, adjusting security controls, or blocking network traffic, based on predefined conditions and incident severity levels." } ] } }, "Databricks": { "The Databricks AI Security Framework": { "link": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirements": [ { "requirementID": "DASF 1: SSO with IdP and MFA", "requirementText": "Implementing single sign-on with an identity provider’s (IdP) multi-factor authentication is critical for secure authentication. It adds an extra layer of security, ensuring that only authorized users access the Databricks Platform." }, { "requirementID": "DASF 2: Sync users and groups\n", "requirementText": "Synchronizing users and groups from your identity provider (IdP) with Databricks using the SCIM standard facilitates consistent and automated user provisioning for enhancing security." }, { "requirementID": "DASF 3: Restrict access using IP access lists", "requirementText": "Configure IP access lists to restrict authentication to Databricks from specific IP ranges, such as VPNs or office networks, and strengthen network security by preventing unauthorized access from untrusted locations. \n\nSecure Egress Gateway (SEG) is a component of Databricks Platform Security that allows an administrator to implement policies that restrict access to internal or external endpoints. Currently, SEG only applies to Serverless runtimes." }, { "requirementID": "DASF 4: Restrict access using private link\n", "requirementText": "Use AWS PrivateLink, Azure Private Link or GCP Private Service Connect to create a private network route between the customer and the Databricks control plane or the control plane and the customer’s compute plane environments to enhance data security by avoiding public internet exposure." }, { "requirementID": "DASF 5: Control access to data and other objects", "requirementText": "Implementing Unity Catalog for unified permissions management and assets simplifies access control and enhances security." }, { "requirementID": "DASF 8: Encrypt data at rest\n", "requirementText": "Databricks supports customer-managed encryption keys to strengthen data at rest protection and greater access control." }, { "requirementID": "DASF 9: Encrypt data in transit", "requirementText": "Databricks supports TLS 1.2+ encryption to protect customer data during transit. This applies to data transfer between the customer and the Databricks control plane and within the compute plane. Customers can also secure inter-cluster communications within the compute plane per their security requirements." }, { "requirementID": "DASF 24: Control access to models and model assets\n", "requirementText": "Organizations commonly encounter challenges in tracking and controlling access to ML models, auditing their usage, and understanding their evolution in complex machine learning workflows. Unity Catalog integrates with the MLflow Model Registry across model lifecycles. This approach simplifies the management and oversight of ML models, proving particularly valuable in environments with multiple teams and diverse projects." }, { "requirementID": "DASF 30: Encrypt models", "requirementText": "Databricks Platform secures model assets and their transfer with TLS 1.2+ in-transit encryption. Additionally, Unity Catalog’s managed model registry provides encryption at rest for persisting models, further enhancing security." }, { "requirementID": "DASF 31: Secure model serving endpoints", "requirementText": "Model serving involves risks of unauthorized data access and model tampering, which can compromise the integrity and reliability of machine learning deployments. Mosaic AI Model Serving addresses these concerns by providing secure-by-default REST API endpoints for MLflow machine learning models, featuring autoscaling, high availability and low latency." }, { "requirementID": "DASF 33: Manage credentials securely", "requirementText": "Databricks Secrets stores your credentials and references them in notebooks, scripts, configuration properties and jobs.\nIntegrating with heterogeneous systems requires managing a potentially large set of credentials and safely distributing them across an organization. Instead of directly entering your credentials into a notebook, use Databricks Secrets to store your credentials and reference them in notebooks and jobs to prevent credential leaks through models. Databricks secret management allows users to use and share credentials within Databricks securely. You can also choose to use a third-party secret management service, such as AWS Secrets Manager or a third-party secret manager." }, { "requirementID": "DASF 34: Run models in multiple layers of isolation", "requirementText": "Databricks Serverless Compute provides a secure-by-design model serving service featuring defense-in-depth controls like dedicated VMs, network segmentation, and encryption for data in transit and at rest. It adheres to the principle of least privilege for enhanced security." }, { "requirementID": "DASF 39: Platform security — Incident Response Team\n", "requirementText": "Databricks has established a formal incident response plan that outlines key elements such as roles, responsibilities, escalation paths and external communication protocols. The platform handles over 9TB of audit logs daily, aiding customer and Databricks security investigations. A dedicated security incident response team operates an internal Databricks instance, consolidating essential log sources for thorough security analysis. Databricks ensures continual operational readiness with a 24/7/365 on-call rotation. Additionally, a proactive hunting program and a specialized detection team support the incident response program and require periodic AI audits and establish protocols for incident reporting, including logs review, performance monitoring, and procedures to report and address misuse." }, { "requirementID": "DASF 40: Platform security — internal access\n", "requirementText": "Databricks personnel, by default, do not have access to customer workspaces or production environments. Access may be temporarily requested by Databricks staff for purposes such as investigating outages, security events or supporting deployments. Customers have the option to disable this access. Additionally, staff activity within these environments is recorded in customer audit logs. Accessing these areas requires multi-factor authentication, and employees must connect to the Databricks VPN." }, { "requirementID": "DASF 43: Use access control lists\n", "requirementText": "Databricks access control lists (ACLs) enable you to configure permissions for accessing and interacting with workspace objects, including folders, notebooks, experiments, models, clusters, pools, jobs, Delta Live Tables pipelines, alerts, dashboards, queries and SQL warehouses." }, { "requirementID": "DASF 46: Store and retrieve embeddings securely", "requirementText": "Mosaic AI Vector Search is a vector database that is built into the Databricks Data Intelligence Platform and integrated with its governance and productivity tools. A vector database is a database that is optimized to store and retrieve embeddings. Embeddings are mathematical representations of the semantic content of data, typically text or image data. Embeddings are usually generated by feature extraction models for text, image, audio or multi-modal data, and are a key component of many GenAI applications that depend on finding documents or images that are similar to each other. Examples are RAG systems, recommender systems, and image and video recognition.\nDatabricks implements the following security controls to protect your data:\nEvery customer request to Vector Search is logically isolated, authenticated and authorized\n\nMosaic AI Vector Search encrypts all data at rest (AES-256) and in transit (TLS 1.2+) and optionally can be encrypted with Customer Managed Keys (CMK). " }, { "requirementID": "DASF 50: Platform compliance", "requirementText": "Develop your solutions on a platform created using some of the most rigorous security and compliance standards in the world. Get independent audit reports verifying that Databricks adheres to security controls for ISO 27001, ISO 27018, SOC 1, SOC 2, FedRAMP, HITRUST, IRAP, etc.\n\nUse platform with established data policy with trust and safety commitments." }, { "requirementID": "DASF 51: Share data and AI assets securely", "requirementText": "Databricks Delta Sharing lets you share data and AI assets securely in Databricks with users outside your organization, whether those users use Databricks or not." }, { "requirementID": "DASF 56: Restrict outbound connections from models", "requirementText": "Egress Control enables you to control outbound connections from your Model Serving compute resources. \nWith this feature, you can restrict access to the internet while allowing access via Unity Catalog Connections or Private Link. Further, this feature blocks direct access to cloud storage (over the shared S3 gateway) to ensure that all data access occurs via Unity Catalog-controlled paths to reduce the risk of data exfiltration." }, { "requirementID": "DASF 57: Use attribute-based access controls (ABAC) \n", "requirementText": "Attribute-based access controls (ABAC) allow data stewards to set policies on data and AI assets using various criteria like user-defined tags, workspace details, location, identity and time. Whether it’s restricting sensitive data to authorized personnel or adjusting access dynamically based on project needs, ABAC ensures security measures are applied with detailed accuracy. Implement attribute-based access controls (ABAC) to define access policies based on attributes or characteristics of the user or the resource being accessed. Use row level filters and column masking for fine-grained access controls." }, { "requirementID": "DASF 60: Rate limit number of inference queries", "requirementText": "Enforce request rate limits to manage traffic at the endpoint level on a per-user and per-endpoint basis, effectively controlling access levels and volume." }, { "requirementID": "DASF 62: Implement network segmentation", "requirementText": "Establish network and security policies to define and enforce egress rules from models and outbound network connections from your serverless compute resources." }, { "requirementID": "DASF 67: Federate authentication", "requirementText": "OAuth token federation is a simpler and more secure method for authenticating to Databricks, especially for automated workloads. Token federation allows applications to authenticate to Databricks using tokens from your trusted IdP, eliminating the need to store Databricks secrets like static tokens or passwords. Databricks OAuth token federation enables your users and applications to securely access Databricks AI APIs using tokens from your identity provider (IdP). Workloads authenticate to Databricks as a service principal in the Databricks account, using workload identity tokens issued by the automation environment. The Databricks SDKs and Databricks CLI automatically fetch these workload identity tokens and exchange them for Databricks OAuth tokens, which eliminates the need manage and rotate Databricks secrets." } ] } }, "ENISA": { "Multilayer Framework for Good Cybersecurity Practices for AI": { "link": "https://www.enisa.europa.eu/sites/default/files/publications/Multilayer%20Framework%20for%20Good%20Cybersecurity%20Practices%20for%20AI.pdf", "requirements": [ { "requirementID": "Model or data disclosure", "requirementText": "Model or data disclosure can be protected by applying proper access control and federated learning to minimise the risk of data breaches. Similarly, to reduce the level of compromise of ML application components, these should be compliant with protection policies, fully integrated to existing security operations and asset management processes, and evaluated according to the level of security of their foundation blocks (e.g. libraries that are responsible for the algorithm implementation). Finally, to prevent failure or malfunction of ML applications, employed algorithms should have their bias reduced, should be properly evaluated to ensure that they are resilient to the environment in which they will operate and should encompass explainability strategies." }, { "requirementID": "Networking 1", "requirementText": "Have you developed / plan to develop national incident management or handling procedures considering AI?" }, { "requirementID": "Networking 3", "requirementText": "Is there collaboration with the national CSIRTs/ CERTs and ISACs for the efficient handling of AI related incidents?" } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.2.2-1", "requirementText": "Developers and System Operators shall evaluate their organization's access control frameworks and identify appropriate measures to secure APIs, models, data, and training and processing pipelines." }, { "requirementID": "Provision 5.2.2-2", "requirementText": "If a Developer offers an API to external customers or collaborators, they shall apply controls that mitigate attacks on the AI system via the API. For example, placing limits on model access rate to limit an attacker's ability to reverse engineer or overwhelm defences to rapidly poison a model." }, { "requirementID": "Provision 5.2.2-3", "requirementText": "Developers shall also create dedicated environments for development and model tuning activities. The dedicated environments shall be backed by technical controls to ensure separation and principle of least privilege. In the context of AI, this is particularly necessary because training data shall only be present in the training and development environments where this training data is not based on publicly available data" }, { "requirementID": "Provision 5.2.2-4", "requirementText": "Developers and System Operators shall implement and publish a clear and accessible vulnerability disclosure policy." }, { "requirementID": "Provision 5.2.2-5", "requirementText": "Developers and System Operators shall create, test and maintain an AI system incident management plan and an AI system recovery plan." }, { "requirementID": "Provision 5.2.2-6", "requirementText": "Developers and System Operators should ensure that, where they are using cloud service operators to help to deliver the capability, their contractual agreements support compliance with the above requirements." } ] }, "SAI 002 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_gr/SAI/001_099/002/01.01.01_60/gr_SAI002v010101p.pdf", "requirements": [ { "requirementID": "6.1.2 Cybersecurity hygiene - 3", "requirementText": "Any keys and passwords used to access data should be secured. Weak passwords and the reuse of compromised passwords are common enterprise security vulnerabilities and apply to both cloud and local storage. A robust password policy and multi-factor authentication should be in place." }, { "requirementID": "6.1.2 Cybersecurity hygiene - 5", "requirementText": "Any organization using cloud storage should understand its responsibilities and the limits of what is provided by its CSP. This is particularly relevant where products move from development into critical operations, and may have inherited risk from the previous research phases." }, { "requirementID": "6.1.2 Cybersecurity hygiene - 6", "requirementText": "A good CI/CD (continuous integration/continuous deployment) pipeline can improve the security of a resultant system, however, tools used in the pipeline should be updated regularly and access to repositories should be monitored [i.11]." }, { "requirementID": "6.1.2 Cybersecurity hygiene - 8", "requirementText": "A cyber incident response plan should be in place and audit processes should be established in order to support analysis of and learning from any security incidents that do take place [i.31]." } ] }, "TR 104 048 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_tr/104000_104099/104048/01.01.01_60/tr_104048v010101p.pdf", "requirements": [ { "requirementID": "6.1.2 Cybersecurity hygiene - 3", "requirementText": "Any keys and passwords used to access data should be secured. Weak passwords and the reuse of compromised passwords are common enterprise security vulnerabilities and apply to both cloud and local storage. A robust password policy and multi-factor authentication should be in place." }, { "requirementID": "6.1.2 Cybersecurity hygiene - 4", "requirementText": "Strong access controls should be in place, applying the principle of least privilege. These stand alongside limits to the number of queries allowed to be made against a model in a period of time." }, { "requirementID": "6.1.2 Cybersecurity hygiene - 5", "requirementText": "Any organization using cloud storage should understand its responsibilities and the limits of what is provided by its CSP. This is particularly relevant where products move from development into critical operations, and may have inherited risk from the previous research phases." }, { "requirementID": "6.1.2 Cybersecurity hygiene - 6", "requirementText": "A good Continuous Integration/Continuous Deployment (CI/CD) pipeline can improve the security of a resultant system, however, tools used in the pipeline should be updated regularly and access to repositories should be monitored." }, { "requirementID": "6.1.2 Cybersecurity hygiene - 8", "requirementText": "A cyber incident response plan should be in place and audit processes should be established in order to support analysis of and learning from any security incidents that do take place." }, { "requirementID": "6.5 Analysis - Following standard cybersecurity good practice", "requirementText": "Following standard cybersecurity good practice, including following the principle of least privilege when accessing data." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.2.2-1", "requirementText": "\"Developers and System Operators shall evaluate their organization's access control frameworks and identify appropriate measures to secure APIs, models, data, and training and processing pipelines.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nInadequate access control can expose sensitive data, models, and pipelines to unauthorized access, increasing the risk of data leaks, model tampering, or unauthorized modifications.\n\nExample Measures/Controls:\nEstablish Role-Based Access Controls (RBAC): Implement role-based access controls to limit access to AI models, data, and pipelines based on user roles and responsibilities, enforcing the principle of least privilege. This includes research environments. Protect API endpoints and data pipelines by implementing access controls, encryption, and authentication mechanisms to prevent unauthorized access." }, { "requirementID": "Provision 5.2.2-2", "requirementText": "\"If a Developer offers an API to external customers or collaborators, they shall apply appropriate controls that mitigate attacks on the AI system via the API. For example, placing limits on model access rate to limit an attacker's ability to reverse engineer or overwhelm defences to rapidly poison a model.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nExternally exposed APIs increase the risk of model extraction attacks, rapid data poisoning, and abuse, potentially compromising the integrity of the AI system.\n\nExample Measures/Controls 1:\nImplement API Rate Limiting: Enforce rate limits on API requests to prevent attackers from overwhelming the system, reverse engineering the model, or rapidly injecting malicious inputs.\n\nExample Measures/Controls 2:\nUse Behavioural Analysis for API Security: Implement behavioural analysis tools to detect abnormal API usage that could indicate malicious intent, such as model extraction or poisoning attempts.\n\nExample Measures/Controls 3:\nDeploy API Gateway with Security Features: Use an API gateway with security features like throttling, dynamic rate limiting, and any other attack detection features, authentication, and logging to manage and monitor access to external-facing APIs." }, { "requirementID": "Provision 5.2.2-3", "requirementText": "\"Developers shall also create dedicated environments for development and model tuning activities. The dedicated environments shall be backed by technical controls to ensure separation and principle of least privilege. In the context of AI, this is particularly necessary because training data shall only be present in the training and development environments where this training data is not based on publicly available data.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout separation and environment-specific controls, production-grade sensitive data and models can be exposed to unauthorized access via development or research workflows, leading to data leaks, tampering, unauthorized deployments, or compliance violations.\n\nExample Measures/Controls:\nSet Up Dedicated Development and Production Environments: Establish separate environments for development, testing, and production, ensuring data and models are only accessible where necessary. Restrict sensitive training data to the development environment, isolating it from production." }, { "requirementID": "Provision 5.2.2-4", "requirementText": "\"Developers and System Operators shall implement and publish a clear and accessible vulnerability disclosure policy.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout a defined vulnerability disclosure policy, security vulnerabilities can go unreported, exposing the organization to delayed or missed opportunities to patch critical issues.\n\nExample Measures/Controls:\nDevelop and Publish a Vulnerability Disclosure Policy: Create a vulnerability disclosure policy that details how vulnerabilities can be reported, including timelines for acknowledgment and resolution." }, { "requirementID": "Provision 5.2.2-5", "requirementText": "\"Developers and System Operators shall create, test, and maintain an AI system incident management plan and an AI system recovery plan.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout a dedicated incident and recovery plan, AI systems can be slow to recover from disruptions, resulting in prolonged downtime or compromised model performance.\n\nExample Measures/Controls:\nDevelop an AI-Specific Incident Management Plan: Create an incident management plan tailored to AI-specific threats, such as discovery of data poisoning, model drift, and adversarial attacks with recovery steps to a known good state, including procedures for validating model integrity." }, { "requirementID": "Provision 5.2.2-6", "requirementText": "\"Developers and System Operators should ensure that, where they are using cloud service operators to help to deliver the capability, their contractual agreements support compliance with the above requirements.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout clear understanding of cloud service agreements, organizations might not know what security measures they can expect or demand from the cloud provider, potentially leaving AI assets exposed to gaps in data protection or compliance failures.\n\nExample Measures/Controls:\nDefine and Validate Security Clauses in Cloud Contracts: Ensure cloud service agreements explicitly outline security responsibilities, compliance standards, and support provisions, including data protection, access controls, incident response, and audit capabilities. Provide detailed documentation to stakeholders to bridge knowledge gaps about the cloud provider's obligations." } ] } }, "European Commission": { "Ethics guidelines for trustworthy AI": { "link": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirements": [ { "requirementID": "1.2.2 Fallback plan and general safety", "requirementText": "AI systems should have safeguards that enable a fallback plan in case of problems. This can mean that AI systems switch from a statistical to rule-based procedure, or that they ask for a human operator before continuing their action. It must be ensured that the system will do what it is supposed to do without harming living beings or the environment." }, { "requirementID": "1.3.3 Access to data", "requirementText": "In any given organisation that handles individuals’ data (whether someone is a user of the system or not), data protocols governing data access should be put in place. These protocols should outline who can access data and under which circumstances. Only duly qualified personnel with the competence and need to access individual’s data should be allowed to do so." } ] } }, "Google": { "Secure AI Framework": { "link": "https://www.saif.google/secure-ai-framework", "requirements": [ { "requirementID": "Application Access Management", "requirementText": "Ensure that only authorized users and endpoints can access specific resources for authorized actions." }, { "requirementID": "Incident Response Management", "requirementText": "Manage response to AI security and privacy incidents." } ] } }, "IBM": { "IBM Framework for Securing Generative AI": { "link": "https://www.ibm.com/products/tutorials/ibm-framework-for-securing-generative-ai", "requirements": [ { "requirementID": "Secure the model", "requirementText": "Within model development, you’re building applications in a new way, and that often involves introducing new, exploitable vulnerabilities that attackers can use as entry points into the environment and, in turn, into your AI models. Considering that organizations have historically struggled with managing a growing debt of known vulnerabilities found within their environments, this risk will carry over to AI.\n\nDeveloping AI applications often starts with data science teams repurposing pretrained, open-source machine learning (ML) models from online model repositories, which often lack comprehensive security controls. However, the value they provide organizations, such as dramatically reducing the time and effort required for generative AI adoption, often outweighs that risk, ultimately passing it on to the enterprise. The general scarcity of security around ML models, coupled with the increasingly sensitive data that ML models are exposed to, means that attacks targeting these models have a high potential for damage.\n\nThe primary attack techniques during model development are supply chain attacks due to the heavy reliance on pretrained, open-source ML models from online model repositories used to accelerate development efforts. Attackers have the same access to these online repositories and can deploy a backdoor or malware into them. Once uploaded back into the repository, they can become an entry point to anyone that downloads the infected model. If these models are infected, it can be incredibly difficult to detect. Organizations must be very cautious about where they consume models and how trusted the source is.\n\nApplication programming interface (API) attacks are another concern. Organizations without the resources or expertise to build their own large language models (LLMs) rely on APIs to consume the capabilities of prepackaged, pretrained models. Attackers recognize this will be a major consumption model for LLMs and will look to target the API interfaces to access and exploit data being transported across the APIs.\n\nAttackers may also seek to exploit LLM agents or plug-ins with excessive permissions to access open-ended functions or downstream systems that can perform privileged actions in business workflows. If an attacker can compromise privileges that are granted to AI agents, the damage could be destructive.\n\nOrganizations’ focus should include:\nContinuously scanning for vulnerabilities, malware and corruption across the AI/ML pipeline\nDiscovering and hardening API and plug-in integrations to third-party models\nConfiguring enforcing policies, controls and RBAC around ML models, artifacts and data sets so that no one person or thing has access to all the data or all of the model functions" }, { "requirementID": "Secure the infrastructure", "requirementText": "One of the first lines of defense is a secure infrastructure. Organizations should leverage existing expertise to optimize security, privacy and compliance standards across distributed environments hosting the AI systems. It’s essential that they harden network security, access control, data encryption, and intrusion detection and prevention around AI environments. They should also consider investing in new security defenses specifically designed to protect AI." } ] } }, "ICO": { "Guidance on the AI Auditing Framework - Draft guidance for consultation ": { "link": "https://ico.org.uk/media2/about-the-ico/consultations/2617219/guidance-on-the-ai-auditing-framework-draft-for-consultation.pdf", "requirements": [ { "requirementID": "Preventative Controls - 8", "requirementText": "Document policy / processes for breach reporting and escalation." }, { "requirementID": "Preventative Controls - 14", "requirementText": "Have an API access policy in place which monitors volume and patterns of requests to identify and report suspicious activity." } ] } }, "IETF": { "Security Requirements for AI Agents": { "link": "https://www.ietf.org/archive/id/draft-ni-a2a-ai-agent-security-requirements-00.html", "requirements": [ { "requirementID": "3.1. Identity Provisioning and Management", "requirementText": "Identity provisioning and management are the process of creating and assigning a verifiable digital identity to an agent.\n\nInitial Trust Establishment: Intial trust can be established through one or more of the following trust anchors, including, but are not limited to: a manufacturer-embedded immutable credential like an IDevID certificate; a hardware root of trust like a Trusted Platform Module (TPM) or Hardware Security Module (HSM); identity documents like an AWS Instance Identity Document or an Azure Managed Service Identity token. This step verifies the agent's execution environment (device, container, etc.) as trustworthy, allows the device or container to join the network, thereby enabling secure operations for all subsequent steps.\n\nCredential Request: During a credential request, the agent must provide multiple proofs of its legitimacy, such as a Certificate Signing Request or a Proof of Possession by signing with the corresponding private key, as well as remote attestation by collecting and submitting evidence to a RATS (Remote Attestation Procedures) Verifier. Additionally, to define the agent's operational scope, the request should incorporate user identity context, binding the credential to a specific human user or an organizational role.\n\nCredential Issuarance: The ACA validates proofs and requests from the above two steps, if passed, it issues an agent-specific credential that may include its owner or requester identity, capabilities, locator, acceptable validation methods for the ARS.\n\nCredential Lifecycle Mangement: The ACA not only issues credentials but also defines and enforces revocation policies. These policies are triggered by specific events, such as a detected security compromise, the agent's scheduled decommissioning, or a key rotation." }, { "requirementID": "3.2. Agent Registration", "requirementText": "After receiving a credential from the ACA, the agent then sends it to the ARS to authenticate itself and start the registration process.\n\nAuthentication: The ARS must verify the legitimacy of the credential submitted by the agent. It must be signed or otherwise endorsed by the ACA.\n\nRegistration: The ARS then checks if the information signed by the ACA, such as the agent's capabilities, exactly matches the registration request sent by the agent. Upon successful validation, the ARS assigns the agent a unique identifier and establishes an agent record that links the identifier to its attributes.\n\nRecord Management: This step automatically removes expired credentials and synchronizes with the ACA to ensure timely revocation of credentials, preventing the use of invalid or compromised credentials." }, { "requirementID": "3.3. Agent Onboarding", "requirementText": "Agent onboarding differs between campus and cloud environments. On campus, agents use protocols like EAP-TLS for network access. In the cloud, the process involves injected sidecars, which register agents to the central service mesh registry automatically to enable communication and management." }, { "requirementID": "4.1. Cross-Domain Identifier Interoperability", "requirementText": "Different domains may use distinct identifier schemas. Possible methods include:\n- pre-configured schema translation\n- cross-domain identifier synchronization\n- a universal parsing framework or system" }, { "requirementID": "4.2. Secure Cross-Domain Transmission", "requirementText": "Mutual TLS (mTLS) connection starts from the external requesting agent to the master agent. The master agent terminates the mTLS connection and parses the application layer requests. In this case, the master agent functions as an OAuth resource server, and manages internal task orchestration." }, { "requirementID": "4.3. Authenticating External Calls", "requirementText": "The master agent then verifies the identity of the requesting agent, and whether or not it has permission to the requested service or agent. Different authentication methods might be possible:\n- API keys\n- Username-password\n- Pre-shared secrets\n- Assertions (for example, JWT Authorization Grant[I-D.draft-ietf-oauth-identity-chaining-06])\nwhich can even be combined with AND/OR logic. During this process, the master agent might be able to identify the caller endpoint type:\n- human user via browser or app\n- human user via API\n- AI agents\n- Hardware or equipment via an IoT API" }, { "requirementID": "4.4. IAM Integration", "requirementText": "Since the agent may inherit its access rights from its owner or user, when authenticating requests, the validation might require integration of IAM systems for redirected verification." }, { "requirementID": "5.1. Authorization Handling", "requirementText": "The master agent acts as the role of OAuth 2.1 resource server. It must validate access tokens as described in OAuth 2.1 Section 5.2. If validation fails, it must respond according to OAuth 2.1 Section 5.3 error handling requirements." }, { "requirementID": "5.2. Authorization Chaining Across Domains", "requirementText": "In an agentic AI use case, a request may traverse multiple resource servers in multiple trust domains before completing. It will be common that the requesting agent from domain A needs to access the resource server (master agent) of domain B. During this process, the following information should be preserved:\nOriginal requesting agent identity\nAuthorization context\n- Scope\n- Resource\n- Audience\n- Grant type\n- Assertion" }, { "requirementID": "0", "requirementText": "Within a domain, there might exist different types of heterogeneous systems or legacy systems that require different authentication methods. They could be API endpoints, microservices, tools or databases. The exact authentication methods are determined by the service itself, for example,\n- bearer tokens\n- API keys\n- pre-shared secrets\n- username-passwords\n- X.509 certificates, etc.\n\nAs a result, the master agent also works as an intermediary credential manager that converts the formats, scopes, identity of the credential, bridging the gap between heterogeneous systems and platforms.\n\nExamples include:\nStatic secrets (API keys) to be exchanged to short-lived, on demand credentials (bearer tokens)" }, { "requirementID": "5.5. Zero Trust Analysis", "requirementText": "The above information can be used as rich context that allow zero trust access control. Remote attestation results of the requesting agent could also be part of access policy decision point's inputs. Remote attestation results of the requesting agent could include the following information:\n- RoT and trust anchors\n- Identifiers\n- Affiliations\n- Posture assessment results\n- Capabilities\n\nThe overall information will be used as input of Policy Engine (PE) and Policy Decision Point (PDP)." }, { "requirementID": "5.6. Microsegmentation", "requirementText": "Microsegmentation may be enforced to prevent lateral movement of security risks. Possible granularity of microsegmentation includes:\n\nper IP segment/subnet\nper each workload\nper tags and attributes (of workload), etc.\n\nThere should be policy enforcement points (PEP) at the perimeter of each segment. Each PEP can receive software-defined security policies issued by PE/PDP." } ] } }, "IMDA": { "Model AI Governance Framework for Agentic AI": { "link": "https://www.imda.gov.sg/-/media/imda/files/about/emerging-tech-and-research/artificial-intelligence/mgf-for-agentic-ai.pdf", "requirements": [ { "requirementID": "2.1.2 Bound risks through design by defining agents limits and permissions - Agent identity", "requirementText": "Identity management and access control is one of the key means in which organisations enable traceability and accountability today for humans. As agents become more autonomous, identity management has to be extended to agents as well to track individual agent behaviour and establish who holds accountability for each agent.\n\nThis is an evolving space, and gaps exist today in terms of handling agent identity robustly. For example, current authorisation systems typically have pre-defined, static scopes. However, to operate safely in more complex scenarios, agents require fine-grained permissions that may change dynamically depending on the context, risk levels, and task objectives. Current authentication systems are also typically based on a single, unique individual. Such systems face difficulty in handling complex agent setups, such as when agents act for multiple human users with different permissions, or recursive delegation scenarios where agents spin up multiple sub-agents.\n\nIn the interim, organisations should consider these best practices to enable agent control and traceability:\n• Identification: An agent should have its own unique identity, such that it can identify itself to the organisation, its human user, or other agents. However, an agent’s identity may need to be tied to a supervising agent, a human user, or an organisational department for accountability and tracking. Additionally, the different capacities in which an agent acts (e.g. independently or on behalf of a specified human user) should also be recorded.\n• Authorisation: An agent can have pre-defined permissions based on its role or the task at hand, or its permissions may be dynamically set by its authorising human user, or a combination of both. As a rule of thumb, the human user should not be able to set permissions for the agent greater than what the human user is himself authorised to do. Such delegations of authority should be clearly recorded." } ] } }, "ISO/IEC": { "DIS 27090": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:27090:dis:ed-1:v1:en", "requirements": [ { "requirementID": "7.6", "requirementText": "Development environment protection" }, { "requirementID": "7.7", "requirementText": "Throttling model use" } ] } }, "Microsoft": { "Cloud Adoption Framework - Secure AI": { "link": "https://learn.microsoft.com/en-us/azure/cloud-adoption-framework/scenarios/ai/secure", "requirements": [ { "requirementID": "Secure AI resources\n2 - Secure all AI communication channels", "requirementText": "Exposed communication paths between AI components allow data interception and system compromise. Properly secured channels prevent unauthorized access and protect sensitive information in transit. Implement managed identities for secure authentication without stored credentials, use virtual networks to isolate AI communications, and deploy Azure API Management to secure Model Context Protocol server endpoints." }, { "requirementID": "Secure AI data\n1 - Define and maintain data boundaries", "requirementText": "Clear data boundaries ensure AI workloads access only data appropriate for their intended audience and use case. Use Microsoft Purview to classify data sensitivity and define access policies. Implement Azure role-based access control (RBAC) to restrict data access by workload and user group. Use Azure Private Link to create network-level data isolation between AI applications." }, { "requirementID": "Detect AI security threats\n2 - Establish AI-focused incident response procedures", "requirementText": "Undetected security incidents can lead to data loss, model compromise, or service disruption that damages business operations. Specialized incident response procedures address the unique characteristics of AI security events. Build and test incident response plans that address AI-specific threats and continuously monitor for indicators of compromise in AI systems. Establish clear escalation procedures for different types of AI security incidents." } ] } }, "MITRE": { "ATLAS Framework": { "link": "https://atlas.mitre.org/mitigations", "requirements": [ { "requirementID": "AML.M0002 - Passive AI Output Obfuscation", "requirementText": "Decreasing the fidelity of model outputs provided to the end user can reduce an adversary's ability to extract information about the model and optimize attacks for the model." }, { "requirementID": "AML.M0004 - Restrict Number of AI Model Queries", "requirementText": "Limit the total number and rate of queries a user can perform." }, { "requirementID": "AML.M0005 - Control Access to AI Models and Data at Rest", "requirementText": "Establish access controls on internal model registries and limit internal access to production models. Limit access to training data only to approved users." }, { "requirementID": "AML.M0007 - Sanitize Training Data", "requirementText": "Detect and remove or remediate poisoned training data. Training data should be sanitized prior to model training and recurrently for an active learning model.\n\nImplement a filter to limit ingested training data. Establish a content policy that would remove unwanted content such as certain explicit or offensive language from being used." }, { "requirementID": "AML.M0015 - Adversarial Input Detection", "requirementText": "Detect and block adversarial inputs or atypical queries that deviate from known benign behavior, exhibit behavior patterns observed in previous attacks or that come from potentially malicious IPs. Incorporate adversarial detection algorithms into the AI system prior to the AI model." }, { "requirementID": "AML.M0016 - Vulnerability Scanning", "requirementText": "Vulnerability scanning is used to find potentially exploitable software vulnerabilities to remediate them.\n\nFile formats such as pickle files that are commonly used to store AI models can contain exploits that allow for arbitrary code execution. These files should be scanned for potentially unsafe calls, which could be used to execute code, create new processes, or establish networking capabilities. Adversaries may embed malicious code in model corrupt model files, so scanners should be capable of working with models that cannot be fully de-serialized. Model artifacts, downstream products produced by models, and external software dependencies should be scanned for known vulnerabilities." }, { "requirementID": "AML.M0017 - AI Model Distribution Methods", "requirementText": "Deploying AI models to edge devices can increase the attack surface of the system. Consider serving models in the cloud to reduce the level of access the adversary has to the model. Also consider computing features in the cloud to prevent gray-box attacks, where an adversary has access to the model preprocessing methods." }, { "requirementID": "AML.M0019 - Control Access to AI Models and Data in Production", "requirementText": "Require users to verify their identities before accessing a production model. Require authentication for API endpoints and monitor production model queries to ensure compliance with usage policies and to prevent model misuse." }, { "requirementID": "AML.M0032 - Segmentation of AI Agent Components", "requirementText": "Define security boundaries around agentic tools and data sources with methods such as API access, container isolation, code execution sandboxing, and rate limiting of tool invocation. This restricts untrusted processes or potential compromises from spreading throughout the system." } ] }, "SAFE-AI": { "link": "https://atlas.mitre.org/pdf-files/SAFEAI_Full_Report.pdf", "requirements": [ { "requirementID": "Insecure APIs", "requirementText": "Insecure APIs can allow attackers unauthorized access, introduce malicious inputs, or disrupt AI systems. This includes risks like unauthorized data access and denial of service, as well as AI-specific threats such as manipulation of model inputs. AI systems often consist of both internal and externally facing APIs that need to be secured, so that data integrity is preserved as data is transmitted among the various components in the AI-enabled system. Standard mitigation strategies, such as data encryption, input validation, robust authentication and authorization mechanisms, are essential for ensuring security of both internal APIs and externally facing APIs. \n\nInference APIs are particularly vulnerable as they are often exposed to external users. Adversaries may exploit legitimate access to inference APIs to gather detailed information about model ontology, structure, and behavior, enabling black-box and white-box attacks. Attackers can refine adversarial techniques to bypass model defenses and evade detection capabilities to introduce malicious data, potentially leading to incorrect predictions or compromised decision-making." }, { "requirementID": "Faulty authentication and authorization settings", "requirementText": "Weak or improperly implemented authentication and authorization mechanisms can allow attackers to poison data, manipulate model input, or otherwise compromise the behavior of an AI component. Note that in AI-enabled systems, faulty settings for these mechanisms may be the result of deliberate attacks like model stealing and prompt extraction, or inadequate attention to data privacy during model development, testing, and deployment." }, { "requirementID": "Insider threats", "requirementText": "Insiders can exploit access privileges to engage in unauthorized activities, including data theft or sabotage of AI models and data. These insider attacks may be especially difficult to address for AI-enabled systems, since AI system development and documentation practices do not tend to employ the same process controls as traditional software development. Moreover, AI systems may require more frequent maintenance and triggers for conducting corrective maintenance due to factors like data, model, or concept drift. This points to the need for extra vigilance when it comes to things like data provenance and access control mechanisms and policies." }, { "requirementID": "Denial of Service attack", "requirementText": "Many AI-enabled systems require significant amounts of specialized computing resources. From an adversary’s perspective, these computing resources can often be viewed as expensive bottlenecks that can be easily overloaded. Adversaries can exploit this vulnerability by flooding the system with inputs, or by intentionally crafting inputs that require heavy amounts of useless compute from the AI system. The increased computing load can eventually degrade or shut down the services supplied by the AIenabled system. Mitigations include limiting the number of queries the AI system will handle at any one time and putting in place explicit monitors to detect adversarial input." }, { "requirementID": "Network components attacks", "requirementText": "The AI components in an AI-enabled system are often general-purpose capabilities that are customized for the needs of the system and its use cases. This means, in particular, that AI-enabled systems often do not have an adequately designed resilient security architecture. There are likely to be shortcomings regarding access controls and proper network configurations. Gaps in these capabilities need to be proactively identified and mitigated during the design, development and deployment phases of the AI lifecycle." }, { "requirementID": "Power supply attacks", "requirementText": "Power supply attacks are problematic for AI-enabled systems that need massive amounts of time and computing resources to process large volumes of complex data during some phase of the AI lifecycle. For example, this is a routine concern for AI pipelines that train modern deep learning systems such as large language models. While the AI architectures supporting these big data requirements are explicitly designed to provide safeguards like checkpoint and restore operations, the I/O bandwidth and storage needed to address these concerns are formidable. The AI concern here is arranging for the massive amount of resources needed for the safeguards, above and beyond what is needed to build and use the AI-enabled system itself." }, { "requirementID": "Physical breaches", "requirementText": "For AI-enabled systems, exploitation of the physical environment to attack the system can occur in a variety of ways. For example, an attacker may physically access the location where data is being collected and modify the collection process in ways that will comprise subsequent AI model training or performance. When the AI system receives input data from real world sensors, it may be possible to employ attacks that make malicious changes to the physical environment that will compromise system behavior (e.g., using physical domain patch-based attack to deceive a ML classification model). Mitigations include stringent approaches to detect physical tampering and unexpected vulnerabilities in AI components. Common anti-tamper technologies for software systems should be applied to AIenabled systems if a physical breach is suspected. Since behavior patterns in AI components can be difficult to specify precisely, it may also be helpful to establish a variety of behavior baselines for AI components. Given a baseline of normal behavior, behavior analysis techniques could identify anomalous behavior patterns that might be useful indicators of tampering." }, { "requirementID": "Identity Spoofing (e.g. deep fakes, synthetic identities, CAPTCHA threat)", "requirementText": "The ability to generate new or altered identities using AI has become common place and represents a threat to some forms of identification and authentication (e.g., voice spoofing. keystroke dynamics, and biometrics). This is particularly true in phishing attacks designed to gain sensitive information that may put access control safeguards at risk. AI based systems can also mimic certain human inputs to break known CAPTCHA-types of systems, thereby increasing the vulnerability to compromise and fraud. Mitigations include stringent access controls, robust authentication and authorization mechanisms, and user education regarding the dangers of social engineering attacks like phishing." }, { "requirementID": "Cost Harvesting", "requirementText": "AI services tend to use large amounts of computing resources and consume a great deal of energy during response generation. Adversaries can maliciously increase the cost of running these services by flooding the system with useless queries, or by crafting computationally expensive inputs. For example, systems that rely on massive neural networks may be vulnerable to adversarial data (e.g., “sponge” examples) designed to activate large numbers of nodes in the hidden network layers." }, { "requirementID": "Spamming the System with Chaff Data", "requirementText": "Adversaries may spam AI applications with chaff data to flood them with false positives, overwhelming the system and increasing detections. This tactic forces analysts to waste time reviewing and correcting incorrect inferences, reducing their efficiency. Techniques include automated scripts, botnets, or tools like Faker to generate large volumes of synthetic data. By inundating the system with irrelevant data, adversaries aim to degrade performance and exhaust the resources." } ] } }, "Model Context Protocol": { "Security Best Practices": { "link": "https://modelcontextprotocol.io/specification/2025-11-25/basic/security_best_practices", "requirements": [ { "requirementID": "Confused Deputy Problem", "requirementText": "Attackers can exploit MCP proxy servers that connect to third-party APIs, creating “confused deputy” vulnerabilities. This attack allows malicious clients to obtain authorization codes without proper user consent by exploiting the combination of static client IDs, dynamic client registration, and consent cookies.\n\nTo prevent confused deputy attacks, MCP proxy servers MUST implement per-client consent and proper security controls as detailed below." }, { "requirementID": "Token Passthrough", "requirementText": "“Token passthrough” is an anti-pattern where an MCP server accepts tokens from an MCP client without validating that the tokens were properly issued to the MCP server and passes them through to the downstream API.\n\nMCP servers MUST NOT accept any tokens that were not explicitly issued for the MCP server." }, { "requirementID": "Server-Side Request Forgery (SSRF)", "requirementText": "Server-Side Request Forgery (SSRF) is an attack where an attacker can induce an MCP client to make HTTP requests to unintended destinations, potentially accessing internal network resources, cloud metadata endpoints, or other protected services.\n\nMCP clients deployed to a server MUST consider SSRF risks and implement appropriate mitigations when fetching OAuth-related URLs. Which protections are appropriate depend on your network environment." }, { "requirementID": "Session Hijacking", "requirementText": "Session hijacking is an attack vector where a client is provided a session ID by the server, and an unauthorized party is able to obtain and use that same session ID to impersonate the original client and perform unauthorized actions on their behalf.\n\nTo prevent session hijacking and event injection attacks, the following mitigations should be implemented:\nMCP servers that implement authorization MUST verify all inbound requests. MCP Servers MUST NOT use sessions for authentication.\n\nMCP servers MUST use secure, non-deterministic session IDs. Generated session IDs (e.g., UUIDs) SHOULD use secure random number generators. Avoid predictable or sequential session identifiers that could be guessed by an attacker. Rotating or expiring session IDs can also reduce the risk.\n\nMCP servers SHOULD bind session IDs to user-specific information. When storing or transmitting session-related data (e.g., in a queue), combine the session ID with information unique to the authorized user, such as their internal user ID. Use a key format like :. This ensures that even if an attacker guesses a session ID, they cannot impersonate another user as the user ID is derived from the user token and not provided by the client.\n\nMCP servers can optionally leverage additional unique identifiers." }, { "requirementID": "Local MCP Server Compromise", "requirementText": "Local MCP servers are MCP Servers running on a user’s local machine, either by the user downloading and executing a server, authoring a server themselves, or installing through a client’s configuration flows. These servers may have direct access to the user’s system and may be accessible to other processes running on the user’s machine, making them attractive targets for attacks.\n\nIf an MCP client supports one-click local MCP server configuration, it MUST implement proper consent mechanisms prior to executing commands." }, { "requirementID": "Scope Minimization", "requirementText": "Poor scope design increases token compromise impact, elevates user friction, and obscures audit trails.\n\nImplement a progressive, least-privilege scope model:\n Minimal initial scope set (e.g., mcp:tools-basic) containing only low-risk discovery/read operations\n Incremental elevation via targeted WWW-Authenticate scope=\"...\" challenges when privileged operations are first attempted\n Down-scoping tolerance: server should accept reduced scope tokens; auth server MAY issue a subset of requested scopes\nServer guidance:\n Emit precise scope challenges; avoid returning the full catalog\n Log elevation events (scope requested, granted subset) with correlation IDs\nClient guidance:\n Begin with only baseline scopes (or those specified by initial WWW-Authenticate)\n Cache recent failures to avoid repeated elevation loops for denied scopes\n" } ] } }, "Multi Agency": { "Guidelines for secure AI system development": { "link": "https://www.ncsc.gov.uk/files/Guidelines-for-secure-AI-system-development.pdf", "requirements": [ { "requirementID": "Secure your infrastructure", "requirementText": "You apply good infrastructure security principles to the infrastructure used in every part of your system’s life cycle. You apply appropriate access controls to your APIs, models and data, and to their training and processing pipelines, in research and development as well as deployment. This includes appropriate segregation of environments holding sensitive code or data. This will also help mitigate standard cyber security attacks which aim to steal a model or harm its performance." }, { "requirementID": "Protect your model continuously", "requirementText": "Attackers may be able to reconstruct the functionality of a model or the data it was trained on, by accessing a model directly (by acquiring model weights) or indirectly (by querying the model via an application or service). Attackers may also tamper with models, data or prompts during or after training, rendering the output untrustworthy.\n\nYou protect the model and data from direct and indirect access, respectively, by:\nImplementing standard cyber security best practices\nImplementing controls on the query interface to detect and prevent attempts to access, modify, and exfiltrate confidential information\n\nTo ensure that consuming systems can validate models, you compute and share cryptographic hashes and/or signatures of model files (for example, model weights) and datasets (including checkpoints) as soon as the model is trained. As always with cryptography, good key management is essential.\n\nYour approach to confidentiality risk mitigation will depend considerably on the use case and the threat model. Some applications, for example those involving very sensitive data, may require theoretical guarantees that can be difficult or expensive to apply. If appropriate, privacy-enhancing technologies (such as differential privacy or homomorphic encryption) can be used to explore or assure levels of risk associated with consumers, users and attackers having access to models and outputs." }, { "requirementID": "Develop incident management procedures", "requirementText": "The inevitability of security incidents affecting your AI systems is reflected in your incident response, escalation and remediation plans. Your plans reflect different scenarios and are regularly reassessed as the system and wider research evolves. You store critical company digital resources in offline backups. Responders have been trained to assess and address AI-related incidents. You provide high-quality audit logs and other security features or information to customers and users at no extra charge, to enable their incident response processes." } ] } }, "NCSC/NSA/CISA etc": { "AI Data Security\n": { "link": "https://media.defense.gov/2025/May/22/2003720601/-1/-1/0/CSI_AI_DATA_SECURITY.PDF", "requirements": [ { "requirementID": "1.4 Leverage trusted infrastructure", "requirementText": "Use a trusted computing environment that leverages Zero Trust architecture. [10] Provide secure enclaves for data processing and keep sensitive information protected and unaltered during computations. This approach fosters a secure foundation for data privacy and security in AI data workflows by isolating sensitive operations and mitigating risks of tampering. Trusted computing infrastructure supports the integrity of data processes, reduces risks associated with unverified or altered data, and ultimately creates a more robust and transparent AI ecosystem. Trusted environments are essential for AI applications where data accuracy directly impacts their decision-making processes." }, { "requirementID": "1.6 Encrypt Data", "requirementText": "Adopt advanced encryption protocols proportional to the organizational data protection level. This includes securing data at rest, in transit, and during processing. AES-256 encryption is the de facto industry standard and is considered resistant to quantum computing threats. [12] [13][13] Use protocols, such as TLS with AES-256 or postquantum encryption, for data in transit. Refer to NIST SP 800-52r2, “Guidelines for the Selection, Configuration, and Use of Transport Layer Security (TLS) Implementations” [14] for more details." }, { "requirementID": "1.7 Store Data Securely", "requirementText": "Store data in certified storage devices that enforce NIST FIPS 140-3 [15] compliance, ensuring that the cryptographic modules used to encrypt the data provide high-level security against advanced intrusion attempts. Note that Security Level 3 (defined in NIST FIPS 140-2 [16]) provides robust data protection; however, evaluate and determine the appropriate level of security based on organizational needs and risk assessments" }, { "requirementID": "2.5 Secure Storage", "requirementText": "After ingest, data needs to be stored in a database that adheres to the best practices for digital signatures, data integrity, and data provenance that are described in detail above. Note that an append-only cryptographically signed database should be used where feasible, but there may be a need to delete older material that is no longer relevant. Each time a data element is updated (e.g., resized, cropped, flipped, etc.) for augmentation purposes in a non-temporary fashion, then the updated data should be stored as a new entry with documented changes. The database’s certificate should be verified at the time the database is accessed for a training run. If the database does not pass the certificate check, abort the training and conduct a comprehensive database audit to discover any data modifications." } ] } }, "NIST": { "AI 100-2e2025: Adversarial Machine Learning\nA Taxonomy and Terminology of Attacks and Mitigations": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-2e2025.pdf", "requirements": [ { "requirementID": "2.3.2 Targeted Poisoning", "requirementText": "Targeted poisoning attacks are notoriously challenging to defend against. Jagielski et al. [180] showed an impossibility result for subpopulation poisoning attacks. To mitigate some of the risks associated with such attacks, model developers may protect training data through traditional cybersecurity measures such as access controls, use methods for data sanitization and validation, and use mechanisms for dataset provenance and integrity attestation [267]. Ma et al. [230] proposed the use of differential privacy (DP) as a defense (which follows directly from the definition of differential privacy), but differentially private ML models may also have lower accuracy than standard models, and the trade-off between robustness and accuracy needs to be considered in each application. See Section 4.1.1 for further discussion on the trade-offs between the attributes of Trustworthy AI systems." }, { "requirementID": "3.3.3 Direct Prompting Attacks - Interventions during deployment (5) - Usage restrictions", "requirementText": "Other interventions have focused on choices about how models are offered to users: for example, the efficacy of some attacks can be reduced by limiting the inference parameters that are accessible to users (e.g., temperature or logit bias), as well as the richness of the model generations returned (e.g., logit probabilities) [250]. Additionally, limiting the release of public information [252, 266] and artifacts [249] and restricting the total number of model queries available to users [251] may make attacks more challenging. These techniques may have additional drawbacksin limiting positive use cases." } ] }, "AI 800-1": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirements": [ { "requirementID": "Practice 3.2: Maintain security practices sufficient to prevent unauthorized access - 3", "requirementText": "Apply appropriate protections against insider threats, such as limiting access to model weights within the organization or implementing two-party control systems." }, { "requirementID": "Practice 4.2: Red-team safeguards - 7", "requirementText": "Consider each level of access to a model that a threat actor might have, ranging from limited access through an API to direct access to the code and parameters that define the model, and determine the minimum level of access (if any) that allows the red team to accomplish its goal." }, { "requirementID": "Practice 6.2: Respond to incidents of model misuse - 1", "requirementText": "Establish clear organizational responsibilities for accountable incident response processes." }, { "requirementID": "Practice 6.2: Respond to incidents of model misuse - 2", "requirementText": "Plan for responses to plausible novel scenarios of misuse, such as strengthening safeguards or implementing new ones." }, { "requirementID": "Practice 6.3: Establish misuse reporting mechanisms - 2", "requirementText": "Establish formal processes to adjudicate concerns from the public, employees, and contractors in a timely fashion." }, { "requirementID": "Practice 6.4: Provide safe harbors for third-party safety research - 1", "requirementText": "Publish a clear vulnerability disclosure policy for model flaws that outlines how such flaws should be shared with the developer and the public, and how the organization will\nrespond to reported flaws." }, { "requirementID": "Practice 6.4: Provide safe harbors for third-party safety research - 2", "requirementText": "Publish a clear safe harbor policy that commits to not pursuing legal action against or restricting the accounts of external safety researchers that act in good faith and comply with the vulnerability disclosure policy." }, { "requirementID": "Practice 6.5: Create bounties for issues related to misuse risk - 1", "requirementText": "Establish a program to incentivize researchers to find model flaws and disclose them according to the vulnerability disclosure policy established in Practice 6.4." }, { "requirementID": "Practice 6.5: Create bounties for issues related to misuse risk - 2", "requirementText": "Consider referring to norms and best practices of existing bug bounty programs, such as those instituted by software vendors, to guide program development." } ] }, "IR 8596: Cybersecurity Framework Profile for Artificial Intelligence (Cyber AI Profile): NIST Community Profile": { "link": "https://csrc.nist.gov/pubs/ir/8596/iprd", "requirements": [ { "requirementID": "ID.IM-04", "requirementText": "Incident response plans and other cybersecurity plans that affect operations are established, communicated, maintained, and improved" }, { "requirementID": "PR.AA-01", "requirementText": "Identities and credentials for authorized users, services, and hardware are managed by the organization" }, { "requirementID": "PR.AA-02", "requirementText": "Identities are proofed and bound to credentials based on the context of interactions" }, { "requirementID": "PR.AA-03", "requirementText": "Users, services, and hardware are authenticated" }, { "requirementID": "PR.AA-04", "requirementText": "Identity assertions are protected, conveyed, and verified" }, { "requirementID": "PR.AA-06", "requirementText": "Physical access to assets is managed, monitored, and enforced commensurate with risk" }, { "requirementID": "PR.DS-04", "requirementText": "Backups of data are created, protected, maintained, and tested" }, { "requirementID": "PR.PS-05", "requirementText": "Installation and execution of unauthorized software are prevented" }, { "requirementID": "PR.IR-01", "requirementText": "Networks and environments are protected from unauthorized logical access and usage" }, { "requirementID": "PR.IR-02", "requirementText": "The organization’s technology assets are protected from environmental threats" }, { "requirementID": "PR.IR-03", "requirementText": "Mechanisms are implemented to achieve resilience requirements in normal and adverse situations " }, { "requirementID": "PR.IR-04", "requirementText": "Adequate resource capacity to ensure availability is maintained" }, { "requirementID": "RS.MA-01", "requirementText": "The incident response plan is executed in coordination with relevant third parties once an incident is declared" }, { "requirementID": "RS.MA-05", "requirementText": "The criteria for initiating incident recovery are applied" }, { "requirementID": "RS.AN-08", "requirementText": "An incident’s magnitude is estimated and validated" }, { "requirementID": "RC.RP-01", "requirementText": "The recovery portion of the incident response plan is executed once initiated from the incident response process" }, { "requirementID": "RC.RP-02", "requirementText": "Recovery actions are selected, scoped, prioritized, and performed" }, { "requirementID": "RC.RP-03", "requirementText": "The integrity of backups and other restoration assets is verified before using them for restoration" }, { "requirementID": "RC.RP-06", "requirementText": "The end of incident recovery is declared based on criteria, and incidentrelated documentation is completed" } ] }, "SP 800-218A": { "link": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirements": [ { "requirementID": "PO.5.2", "requirementText": "Secure and harden development endpoints (endpoints for software designers, developers, testers, builders, etc.) to perform development tasks using a risk-based approach." }, { "requirementID": "PW.1.3", "requirementText": "Where appropriate, build in support for using standardized security features and services (e.g., enabling software to integrate with existing log management, identity management, access control, and vulnerability management systems) instead of creating proprietary implementations of security features and services." }, { "requirementID": "RV.1.3", "requirementText": "Have a policy that addresses vulnerability disclosure and remediation, and implement the roles, responsibilities, and processes needed to support that policy.\n\nInclude AI model vulnerabilities in organization vulnerability disclosure and remediation policies.\n\nMake users of AI models aware of their inherent limitations and how to report any cybersecurity problems that they encounter." } ] } }, "OWASP": { "LLM Top 10": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirements": [ { "requirementID": "LLM02: Sensitive Information Disclosure - 3", "requirementText": "Limit access to sensitive data based on the principle of least privilege. Only grant access to data that is necessary for the specific user or process." }, { "requirementID": "LLM02: Sensitive Information Disclosure - 5", "requirementText": "Train models using decentralized data stored across multiple servers or devices. This approach minimizes the need for centralized data collection and reduces exposure risks.\n" }, { "requirementID": "LLM04: Data and Model Poisoning - 3", "requirementText": "Implement strict sandboxing to limit model exposure to unverified data sources. Use anomaly detection techniques to filter out adversarial data." }, { "requirementID": "LLM04: Data and Model Poisoning - 5", "requirementText": "Ensure sufficient infrastructure controls to prevent the model from accessing unintended data sources." }, { "requirementID": "LLM05: Improper Data Handling - 5", "requirementText": "Use parameterized queries or prepared statements for all database operations involving LLM output." }, { "requirementID": "LLM05: Improper Data Handling - 6", "requirementText": "Employ strict Content Security Policies (CSP) to mitigate the risk of XSS attacks from LLMgenerated content." }, { "requirementID": "LLM07: System Prompt Leakage - 1", "requirementText": "Avoid embedding any sensitive information (e.g. API keys, auth keys, database names, user roles, permission structure of the application) directly in the system prompts. Instead, externalize such information to the systems that the model does not directly access." }, { "requirementID": "LLM07: System Prompt Leakage - 2", "requirementText": "Since LLMs are susceptible to other attacks like prompt injections which can alter the system prompt, it is recommended to avoid using system prompts to control the model behavior where possible. Instead, rely on systems outside of the LLM to ensure this behavior. For example, detecting and preventing harmful content should be done in external systems." }, { "requirementID": "LLM07: System Prompt Leakage - 3", "requirementText": "Implement a system of guardrails outside of the LLM itself. While training particular behavior into a model can be effective, such as training it not to reveal its system prompt, it is not a guarantee that the model will always adhere to this. An independent system that can inspect the output to determine if the model is in compliance with expectations is preferable to system prompt instructions." }, { "requirementID": "LLM07: System Prompt Leakage - 4", "requirementText": "Critical controls such as privilege separation, authorization bounds checks, and similar must not be delegated to the LLM, either through the system prompt or otherwise. These controls need to occur in a deterministic, auditable manner, and LLMs are not (currently) conducive to this. In cases where an agent is performing tasks, if those tasks require different levels of access, then multiple agents should be used, each configured with the least privileges needed to perform the desired tasks." }, { "requirementID": "LLM08: Vector and Embedding Weaknesses - 1", "requirementText": "Implement fine-grained access controls and permission-aware vector and embedding stores. Ensure strict logical and access partitioning of datasets in the vector database to prevent unauthorized access between different classes of users or different groups." }, { "requirementID": "LLM10: Unbounded Consumption - 3", "requirementText": "Apply rate limiting and user quotas to restrict the number of requests a single source entity can make in a given time period." }, { "requirementID": "LLM10: Unbounded Consumption - 4", "requirementText": "Monitor and manage resource allocation dynamically to prevent any single user or request from consuming excessive resources." }, { "requirementID": "LLM10: Unbounded Consumption - 5", "requirementText": "Set timeouts and throttle processing for resource-intensive operations to prevent prolonged resource consumption." }, { "requirementID": "LLM10: Unbounded Consumption - 10", "requirementText": "Implement restrictions on the number of queued actions and total actions, while incorporating dynamic scaling and load balancing to handle varying demands and ensure consistent system performance." }, { "requirementID": "LLM10: Unbounded Consumption - 13", "requirementText": "Implement strong access controls, including role-based access control (RBAC) and the principle of least privilege, to limit unauthorized access to LLM model repositories and training environments." }, { "requirementID": "LLM10: Unbounded Consumption - 15", "requirementText": "Implement automated MLOps deployment with governance, tracking, and approval workflows to tighten access and deployment controls within the infrastructure." } ] }, "OWASP Model Context Protocol (MCP) Top 10": { "link": "https://owasp.org/www-project-mcp-top-10/", "requirements": [ { "requirementID": "MCP04:2025 – Software Supply Chain Attacks & Dependency Tampering - 4", "requirementText": "Version Pinning & Approved Registries\n- Pin component versions — avoid “latest”\n- Use internal package mirrors or registries\n- Block direct downloads from the public internet" }, { "requirementID": "MCP04:2025 – Software Supply Chain Attacks & Dependency Tampering - 6", "requirementText": "Sandbox Third-Party Plugins\n- Run plugins in constrained environments (e.g., WASM, container isolation)\n- Restrict filesystem + network access" }, { "requirementID": "MCP05:2025 – Command Injection & Execution - 3", "requirementText": "Sandbox All Tools\n- Run tools inside containers, micro-VMs, gVisor/Kata, or jailed users.\n- Enforce timeouts, resource limits, and read-only file systems. Isolate high-risk tools (file system, network, DB) into separate sandboxes." }, { "requirementID": "MCP05:2025 – Command Injection & Execution - 5", "requirementText": "Strong Validation at Tool Boundaries Validate agent output against schemas before execution. Use parameterized SQL/APIs — never interpolate input. Reject unsafe patterns: chained commands, redirection, wildcards, command substitution." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 1", "requirementText": "Strong Authentication for All Entities\n- Require mutual TLS (mTLS) between MCP clients, agents, and servers.\n- Use short-lived, scoped tokens (JWT/OAuth2-style) tied to specific sessions and permissions.\n- Enforce token binding to agent identity (e.g., signed agent attestation).\n- Validate every token on the server side — never trust client-provided claims." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 2", "requirementText": "Implement Fine-Grained Authorization\n- Adopt RBAC (roles) or ABAC (attributes) models: Example: “Agent X may read customer data but not execute tools.”\n- Evaluate permissions per request, not per session.\n- Deny-by-default: any unrecognized agent or scope should be blocked automatically." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 3", "requirementText": "Token Lifecycle Management\n- Enforce expiration, rotation, and revocation policies for all tokens.\n- Store tokens securely (vaulted or encrypted).\n- Detect and block replayed or duplicated tokens." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 5", "requirementText": "Centralized Identity & Access Management\n- Integrate MCP authentication with organizational IAM or OIDC providers.\n- Require federated identity for all user-driven and system-driven actions.\n- Centralize policy enforcement through a Policy Decision Point (PDP)." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 7", "requirementText": "Secure-by-Default Configurations\n- Disable guest or anonymous access in all MCP endpoints.\n- Prevent local testing servers from exposing endpoints publicly.\n- Enforce environment-specific credentials for dev/test/prod." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 8", "requirementText": "Revoke all compromised or static tokens immediately." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 9", "requirementText": "Rotate all service credentials and enforce unique per-agent identities." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 10", "requirementText": "Enable mTLS and strict API key binding." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 13", "requirementText": "Add temporary compensating controls: IP restrictions, manual approvals for sensitive actions." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 1", "requirementText": "Establish Central MCP Governance & Registry\n- Create a centralized MCP registry where every instance must be registered before deployment.\n- Tie registration to CI/CD pipelines — any unregistered instance should fail deployment.\n- Maintain metadata: owner, purpose, version, endpoints, compliance state, and contact.\n- Require approval and risk classification for each new MCP instance." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 2", "requirementText": "Implement Discovery & Continuous Scanning\n- Use network discovery tools (Nmap, Shodan internal equivalents, CSPM, or EASM tools) to detect open MCP ports and endpoints.\n- Deploy passive network sensors to identify MCP traffic patterns (unique protocol identifiers, routes).\n- Integrate discovery results with asset inventories and vulnerability management platforms.\n- Automate shadow MCP detection scans weekly with alerts to the security operations team." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 3", "requirementText": "Define Baseline Configuration Templates\n- Publish secure-by-default MCP configuration templates for teams:\n- Enforce authentication and authorization (mTLS, OAuth).\n- Disable unauthenticated tool calls and external access by default.\n- Include preconfigured logging, rate-limits, and monitoring agents.\n- Block deployment of MCP instances that deviate from approved templates." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 4", "requirementText": "Enforce Identity & Access Management (IAM) Controls\n- Require all MCP instances to integrate with central IAM providers (SSO, LDAP, or OIDC).\n- Use service identities bound to teams and enforce role-based access.\n- Apply network segmentation (VPC-level controls, firewall rules) to limit exposure." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 7", "requirementText": "Policy & Enforcement\n- Integrate MCP governance into corporate IT and AI Acceptable Use Policies (AUPs).\n- Require sign-off from information security before deployment of any model-serving or context protocol infrastructure.\n- Periodically audit compliance and enforce disciplinary or procedural action for unauthorized setups." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 9", "requirementText": "Contain the detected shadow MCP (disable network access, snapshot for forensics)." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 10", "requirementText": "Identify owners and isolate associated credentials or API keys." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 13", "requirementText": "Enforce registration and compliance checks before re-enabling access." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 14", "requirementText": "Update network segmentation and discovery coverage to prevent recurrence." }, { "requirementID": "MCP10:2025 – Context Injection & Over-Sharing - 1", "requirementText": "Use Ephemeral Contexts\n- Make context windows short-lived and per session by default.\n- Enforce automatic deletion after task completion.\n- Avoid persistent memory unless explicitly sanctioned and governed." }, { "requirementID": "MCP10:2025 – Context Injection & Over-Sharing - 4", "requirementText": "Context Expiry and TTL Enforcement\nDefine time-to-live (TTL) policies such as:\nSession end\n30 minutes\n24 hours max Automatically purge expired contexts and embeddings." } ] }, "OWASP Top 10 for Agentic Applications for 2026": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "requirements": [ { "requirementID": "ASI01: Agent Goal Hijack - 2", "requirementText": "Minimize the impact of goal hijacking by enforcing least privilege for agent tools and requiring human approval for high-impact or goal-changing actions." }, { "requirementID": "ASI01: Agent Goal Hijack - 3", "requirementText": "Define and lock agent system prompts so that goal priorities and permitted actions are explicit and auditable. Changes to changes in goals or reward definitions must go through configuration management and human approval." }, { "requirementID": "ASI01: Agent Goal Hijack - 4", "requirementText": "At run time, validate both user intent and agent intent before executing goal-changing or high-impact actions. Require confirmation - via human approval, policy engine, or platform guardrails whenever the agent proposes actions that deviate from the original task or scope. Pause or block execution on any unexpected goal shift, surface the deviation for review, and record it for audit." }, { "requirementID": "ASI01: Agent Goal Hijack - 9", "requirementText": "Incorporate AI Agents into the established Insider Threat Program to monitor any insider prompts intended to get access to sensitive data or to alter the agent behavior and allow for investigation in case of outlier activity." }, { "requirementID": "ASI02: Tool Misuse and Exploitation - 3", "requirementText": "Execution Sandboxes and Egress Controls. Run tool or code execution in isolated sandboxes. Enforce outbound allowlists and deny all non-approved network destinations." }, { "requirementID": "ASI02: Tool Misuse and Exploitation - 4", "requirementText": "Policy Enforcement Middleware (“Intent Gate”). Treat LLM or planner outputs as untrusted. A pre-execution Policy Enforcement Point (PEP/PDP) validates intent and arguments, enforces schemas and rate limits, issues short-lived credentials, and revokes or audits on drift." }, { "requirementID": "ASI02: Tool Misuse and Exploitation - 5", "requirementText": "Adaptive Tool Budgeting. Apply usage ceilings (cost, rate, or token budgets) with automatic revocation or throttling when exceeded." }, { "requirementID": "ASI02: Tool Misuse and Exploitation - 6", "requirementText": "Just-in-Time and Ephemeral Access. Grant temporary credentials or API tokens that expire immediately after use. Bind keys to specific user sessions to prevent lateral abuse." }, { "requirementID": "ASI02: Tool Misuse and Exploitation - 7", "requirementText": "Semantic and Identity Validation (‘Semantic Firewalls)”. Enforce fully qualified tool names and version pins to avoid tool alias collisions or typo squatted tools; validate the intended semantics of tool calls (e.g., query type or category) rather than relying on syntax alone. Fail closed on ambiguous resolution and prompt for user disambiguation." }, { "requirementID": "ASI04: Agentic Supply Chain Vulnerabilities - 3", "requirementText": "Containment and builds: Run sensitive agents in sandboxed containers with strict network or syscall limits; require reproducible builds." }, { "requirementID": "ASI04: Agentic Supply Chain Vulnerabilities - 5", "requirementText": "Inter-agent security: Enforce mutual auth and attestation via PKI and mTLS; no open registration; sign and verify all inter-agent messages." }, { "requirementID": "ASI05: Unexpected Code Execution (RCE) - 4", "requirementText": "Execution environment security: Never run as root. Run code in sandboxed containers with strict limits including network access; lint and block known-vulnerable packages and use framework Page 23genai.owasp.org sandboxes like mcp-run-python. Where possible, restrict filesystem access to a dedicated working directory and log file diffs for critical paths." }, { "requirementID": "ASI05: Unexpected Code Execution (RCE) - 5", "requirementText": "Architecture and design: Isolate per-session environments with permission boundaries; apply least privilege; fail secure by default; separate code generation from execution with validation gates." }, { "requirementID": "ASI06: Memory & Context Poisoning - 1", "requirementText": "Baseline data protection: Encryption in transit and at rest combined with least-privilege access." }, { "requirementID": "ASI06: Memory & Context Poisoning - 3", "requirementText": "Memory segmentation: Isolate user sessions and domain contexts to prevent knowledge and sensitive data leakage." }, { "requirementID": "ASI06: Memory & Context Poisoning - 4", "requirementText": "Access and retention: Allow only authenticated, curated sources; enforce context-aware access per task; minimize retention by data sensitivity." }, { "requirementID": "ASI07: Insecure Inter-Agent Communication - 1", "requirementText": "Secure agent channels: Use end-to-end encryption with per-agent credentials and mutual authentication. Enforce PKI certificate pinning, forward secrecy, and regular protocol reviews to prevent interception or spoofing." }, { "requirementID": "ASI07: Insecure Inter-Agent Communication - 2", "requirementText": "Message integrity and semantic protection: Digitally sign messages, hash both payload and context, and validate for hidden or modified natural-language instructions. Apply natural-language– aware sanitization and intent-diffing to detect goal, parameter tampering, hidden or modified natural-language instructions." }, { "requirementID": "ASI07: Insecure Inter-Agent Communication - 3", "requirementText": "Agent-aware anti-replay: Protect all exchanges with nonces, session identifiers, and timestamps tied to task windows. Maintain short-term message fingerprints or state hashes to detect cross-context replays." }, { "requirementID": "ASI07: Insecure Inter-Agent Communication - 4", "requirementText": "Protocol and capability security: Disable weak or legacy communication modes. Require agent-specific trust negotiation and bind protocol authentication to agent identity. Enforce version and capability policies at gateways or middleware." }, { "requirementID": "ASI07: Insecure Inter-Agent Communication - 5", "requirementText": "Limit metadata-based inference: Reduce the attack surface for traffic analysis by using fixed-size or padded messages where feasible, smoothing communication rates, and avoiding deterministic communication schedules. These lightweight measures make it harder for attackers to infer agent roles or decision cycles from metadata alone, without requiring heavy protocol redesign." }, { "requirementID": "ASI07: Insecure Inter-Agent Communication - 6", "requirementText": "Protocol pinning and version enforcement: Define and enforce allowed protocol versions (e.g., MCP, A2A, gRPC). Reject downgrade attempts or unrecognized schemas and validate that both peers advertise matching capability and version fingerprints." }, { "requirementID": "ASI07: Insecure Inter-Agent Communication - 7", "requirementText": "Discovery and routing protection. Authenticate all discovery and coordination messages using cryptographic identity. Secure directories with access controls and verified reputations, validate identity and intent end-to-end, and monitor for anomalous routing flows." }, { "requirementID": "ASI08: Cascading Failures - 2", "requirementText": "Isolation and trust boundaries: Sandbox agents, least privilege, network segmentation, scoped APIs, and mutual auth. to contain failure propagation." }, { "requirementID": "ASI08: Cascading Failures - 3", "requirementText": "JIT, one-time tool access with runtime checks: Issue short-lived, task-scoped credentials for each agent run and validate every high-impact tool invocation against a policy-as-code rule before executing it. This ensures a compromised or drifting agent cannot trigger chain reactions across other agents or systems." }, { "requirementID": "ASI08: Cascading Failures - 6", "requirementText": "Rate limiting and monitoring: Detect fast-spreading commands and throttle or pause on anomalies." }, { "requirementID": "ASI08: Cascading Failures - 7", "requirementText": "Implement blast-radius guardrails such as quotas, progress caps, circuit breakers between planner and executor." }, { "requirementID": "ASI09: Human-Agent Trust Exploitation - 6", "requirementText": "Content provenance and policy enforcement: Attach verifiable metadata-source identifiers, timestamps, and integrity hashes-to all recommendations and external data. Enforce digital signature validation and runtime policy checks that block actions lacking trusted provenance or exceeding the agent’s declared scope." }, { "requirementID": "ASI09: Human-Agent Trust Exploitation - 7", "requirementText": "Separate preview from effect: Block any network or state-changing calls during preview context and display a risk badge with source provenance and expected side effects." }, { "requirementID": "ASI10: Rogue Agents - 2", "requirementText": "Isolation & Boundaries: Assign Trust Zones with strict inter-zone communication rules and deploy restricted execution environments (e.g., container sandboxes) with API scopes based on least privilege." }, { "requirementID": "ASI10: Rogue Agents - 6", "requirementText": "Require periodic behavioral attestation: challenge tasks, signed bill of materials for prompts and tools, and per-run ephemeral credentials with one-time audience binding. All signing and attestation mechanisms assume hardened cryptographic key management (e.g., HSM/KMS-backed keys, least- Page 38genai.owasp.org privilege access, rotation and revocation). Keys must never be directly available to agents; instead, orchestrators should mediate signing operations so that a compromised agent cannot simply exfiltrate or misuse long-lived keys" } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirements": [ { "requirementID": "13.6.2", "requirementText": "The Al System must have built in guard rails or specific limits that the Al cannot override." }, { "requirementID": "13.6.3", "requirementText": "Entities must review the guard rails and specific limits on a frequent set schedule or in response to external volatility spikes." } ] } }, "SANS": { "Critical AI Security Guidelines": { "link": "https://sansorg.egnyte.com/dl/bvkYQxrW8QMj", "requirements": [ { "requirementID": "1.1 Protect Your Model Parameters", "requirementText": "It is critical to ensure traditional security controls, such as principle of least privilege and strong access controls with accountability, have been implemented. Should an unauthorized individual be able to replace or modify a deployed model, untold damage can result. Although this applies to any kind of AI model developed and deployed by an enterprise—including training models, which are highly susceptible to poisoning and attacks—consider the example of a generative agentic system leveraging a large language model (LLM)" }, { "requirementID": "3.2 AI Deployment in Integrated Development Environments (IDEs)", "requirementText": "IDEs such as VSCode, Windsurf, or Cursor are fully integrated with models or offer LLM integration as a highly desirable option. Although these integrations can significantly increase the efficiency and output of developers, users can inadvertently expose proprietary algorithms, models, API keys, and datasets through AI-powered features. Organizations should explore IDEs with local-only LLM integrations to mitigate risk exposure when local-only LLM integration becomes available. This control ensures that sensitive data remains secure and protected." }, { "requirementID": "4.9 Control and Monitor Access to Interaction/Inference", "requirementText": "Depending on the use case and deployment of the LLM application, authentication and access controls should be implemented where appropriate. For public-facing applications, such as help or chatbots to aid or guide website visitors, there is no need to require authentication for user interaction. For external-facing applications, the front-end must authenticate to the LLM back-end, and as such, these calls should have audit and access logging enabled. Similarly, internal LLM applications, or those containing sensitive data, should be used with authentication and access controls, with auditing enabled by default in an enterprise environment. The ability to interact with an LLM application and have the model perform inference should be restricted, according to business use cases. Unless absolutely necessary, unauthenticated and/or unmonitored access to LLM APIs or front-ends should not be allowed." }, { "requirementID": "4.10 Monitor/Control API Usage", "requirementText": "Abuse of LLMs can occur via multiple means. Prompt injection protection and detection methods primarily focus on the content of the input. Content validation, monitoring, and filtering also should be complemented with usage and behavior monitoring focused on the interactions themselves. LLM API keys should be properly managed under robust, secure software development policies, such as no hardcoding of keys in applications. Observing API usage for misuse is critical. Anomalous spikes in API usage can serve as an effective detection method for abuse, while rate limiting should be considered to restrict the number and cadence of interactions allowed. In addition to rate limiting, organizations also should consider other forms of behavior/anomaly detection. Although not limited to interactions through APIs, adversaries can easily automate inputs to exposed API endpoints, making them more susceptible to volumetric attacks. To mitigate this risk, internal training or inference API endpoints should not be public facing." } ] } }, "SDAIA (Saudi Arabia)": { "AI Adoption Framework": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Files/AIAdoptionFramework.pdf", "requirements": [ { "requirementID": "5.1.2 Privacy and Safety - Activating Recovery and Disaster Plans", "requirementText": "Implementing professional recovery plans helps mitigate the impact of unexpected outages. These plans include automated backups, ready-to-launch backup sites, and automated response mechanisms that restore systems within minutes, ensuring business continuity and protecting critical data." } ] }, "AI Ethics Principles": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Documents/ai-principles.pdf", "requirements": [ { "requirementID": "Principle 2 – Privacy & Security - Prepare Input Data - 2", "requirementText": "The confidentiality of data ensures that information is accessible only to those who are authorized to access the information and that there are specific controls that manage the delegation of authority." } ] } } }, "Principle 7": { "Atlantic Council": { "Securing data in the AI supply chain": { "link": "https://www.atlanticcouncil.org/in-depth-research-reports/issue-brief/securing-data-in-the-ai-supply-chain/", "requirements": [ { "requirementID": "Recommendation 1", "requirementText": "Developers, users, maintainers, governors, and securers of AI technologies should map the data components of the AI supply chain to existing cybersecurity best practices—and use that mapping to identify where existing best practices fall short for AI-specific risks to the data components of the AI supply chain." }, { "requirementID": "Recommendation 2", "requirementText": "Developers, users, maintainers, governors, and securers of AI technologies should “Know Your Supplier,” using the supply chain-focused approach to mitigate both AI-specific and non-AI-specific risks to the data components of the AI supply chain." }, { "requirementID": "Recommendation 3", "requirementText": "Policymakers should widen their lens on AI data to encompass all data components of the AI supply chain. This includes assessing whether sufficient attention is given to the diversity of data use cases that need protection (e.g., not just training data for chatbots but for transportation safety or drug discovery) and whether they have mapped existing security best practices to non-AI-specific and AI-specific risks." } ] } }, "Central Bank of the UAE": { "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E": { "link": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirements": [ { "requirementID": "9. Outsourcing and Third-Party Risk - a", "requirementText": "Where LFIs rely on third-party vendors or cloud service providers for AI and ML models, products or solutions, due diligence should be conducted on the provider’s reputation in the field of AI, governance, security and data-protection practices. Contracts should include provisions that ensure access to relevant information, audit rights and compliance with CBUAE requirements." }, { "requirementID": "9. Outsourcing and Third-Party Risk - b", "requirementText": "The outsourcing of any AI system and/or engagement with third party providers of AI should only be carried out with appropriate diligence, assessment and it should be considered if only to be done by way of Board of Director and/or Senior Management approval, with such processes being appropriately documented for audit purposes. " }, { "requirementID": "9. Outsourcing and Third-Party Risk - c", "requirementText": "The procurement, choice and justification for selection of a third-party AI provider and deployment of their AI models/systems/technology at an LFI should be appropriate and documented, which should include annual cybersecurity reviews by independent and suitably qualified third parties and pre-deployment tests and checks to ensure appropriateness and taking into accounts needs, risks and the options available. " } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirements": [ { "requirementID": "1.2.2 Secure Procurement or Development", "requirementText": "Select vendors who adhere to secure practices and develop AI systems using secure methodologies and tools." }, { "requirementID": "1.2.5 Procure an AI System", "requirementText": "Critical infrastructure owners and operators should also carefully evaluate the trade-offs between different methods for sourcing an AI system" }, { "requirementID": "2.2.4 - Data Privacy and Security", "requirementText": "Protect proprietary and personal information within OT datasets, including by instituting protection from access abuse, intentional or inadvertent data poisoning, or dependency on synthetic, generated data." }, { "requirementID": "2.3 Understanding the role of OT Vendors in AI Integration", "requirementText": "Critical infrastructure owners and operators should demand transparency and security considerations from OT vendors regarding how AI technologies are embedded into their products" } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "MDS-02", "requirementText": "Define, implement, and evaluate policies, procedures, and technical measures for the scanning of model artifacts for vulnerabilities and attacks, at each step of the service lifecycle and at each hand over point. Regularly review and update policies, procedures and technical measures to address model artifact scanning." }, { "requirementID": "STA-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate, and maintain policies and procedures for supply chain risk management. Review and update the policies and procedures at least annually or upon significant changes." }, { "requirementID": "STA-09", "requirementText": "Periodically review risk factors associated with supply chain relationships." }, { "requirementID": "STA-11", "requirementText": "Review supply chain agreements at least annually, or upon significant changes." }, { "requirementID": "STA-13", "requirementText": "Implement policies requiring all service providers throughout the supply chain to comply with information security, confidentiality, access control, privacy, audit, personnel policy and service level requirements and standards." }, { "requirementID": "STA-14", "requirementText": "Periodically review the organization's supply chain partners' IT governance policies and procedures." }, { "requirementID": "STA-16", "requirementText": "Define, implement, and enforce a process for establishing a Bill of Material for the service supply chain. Review and update the Bill of Material at least annually or upon significant changes." }, { "requirementID": "UEM-14", "requirementText": "Define, implement and evaluate processes, procedures and technical and/or contractual measures to maintain proper security of third-party endpoints with access to organizational assets." } ] } }, "CoSAI": { "Establish Risks and Controls for the AI Supply Chain": { "link": "https://github.com/cosai-oasis/ws1-supply-chain/blob/main/risks-and-controls-for-the-ai-supply-chain-v1.md", "requirements": [ { "requirementID": "3.1.2 Model Training - Vulnerable Dependency", "requirementText": "Training pipeline dependencies containing security vulnerabilities:\nComprehensive supply chain visibility, regular dependency updates, dependency documentation for each training run" }, { "requirementID": "3.1.2 Model Training - ML Framework Vulnerability", "requirementText": "Security flaws in the core ML framework used for training or inference:\nRegular framework updates, version documentation in supply chain records" }, { "requirementID": "3.1.3 Application Integration in the AI Supply Chain - Compromised Dependencies", "requirementText": "Malicious code insertion or vulnerability exploitation in application libraries:\nComprehensive dependency validation, SBOMs, automated scanning, verified builds" }, { "requirementID": "3.1.3 Application Integration in the AI Supply Chain - Abandoned Dependencies", "requirementText": "Security vulnerabilities in unmaintained application components:\nDependency health monitoring, lifecycle policies, replacement strategies" }, { "requirementID": "3.1.4 AI Infrastructure Supply Chain - Third-Party Dependency Compromise", "requirementText": "Supply chain attacks targeting dependencies of the hosting infrastructure:\nReview dependency management practices, request SBOM for infrastructure components" }, { "requirementID": "3.2.3 Application - Vulnerability in MLOps Tooling or ML Libraries", "requirementText": "Security defects present in development, deployment, or monitoring components within the AI infrastructure stack that can be exploited to compromise model integrity or operational security. These vulnerabilities often exist in underlying dependencies rather than primary components:\nImplement systematic update management for all libraries and MLOps tooling with comprehensive vulnerability scanning. Document all tools and libraries in standardized supply chain artifacts for each training run to enable rapid impact assessment when vulnerabilities are discovered." } ] }, "Model Context Protocol (MCP) Security": { "link": "https://github.com/cosai-oasis/ws4-secure-design-agentic-systems/blob/main/model-context-protocol-security.md", "requirements": [ { "requirementID": "3.2.11 Lifecycle and Governance b)", "requirementText": "Supply chain security requires:\n- implementing software bill of materials (SBOM) tracking for all MCP components,\n- using dependency pinning with hash-based verification rather than version ranges,\n- deploying automated vulnerability scanning for MCP servers and dependencies,\n- implementing secure software development lifecycle (SSDLC) practices for internal MCP servers,\n- using reproducible builds to verify package authenticity,\n- and monitoring security advisories and CVE databases for known vulnerabilities in dependencies." } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "2.1.3", "requirementText": "The entity shall identify, assess, and manage cyber security risks associated with the supply chain of AI/ML systems, ensuring appropriate controls are implemented to mitigate threats arising from third-party components, services, and dependencies." }, { "requirementID": "2.2.1", "requirementText": "The entity shall maintain an up-to-date inventory of AI/ML assets and apply cyber security controls commensurate with the sensitivity, criticality, and value of each asset." }, { "requirementID": "3.1.3 AI Supply Chain Security - 1", "requirementText": "The entity should establish a comprehensive AI supply chain security strategy to manage cyber security risks associated with third-party AI/ML components and services, from sourcing to decommissioning." }, { "requirementID": "3.1.3 AI Supply Chain Security - 2", "requirementText": "The entity should ensure a clear understanding of its AI/ML supply chain, including the source of components, models, training data, and libraries used in its AI/ML systems." }, { "requirementID": "3.1.3 AI Supply Chain Security - 4", "requirementText": "The entity should conduct regular security assessments of its AI/ML supply chain, including vendors, developers, and other third parties contributing to the AI/ML system. The entity may alternatively leverage independent third-party assurance reports (e.g., SOC 2, ISO/IEC 27001) or recognized certifications to inform its assessment." }, { "requirementID": "3.1.3 AI Supply Chain Security - 5", "requirementText": "The entity should ensure that contractual agreements with vendors and other third parties address cyber security requirements for AI/ML systems." }, { "requirementID": "3.2.1 Asset Management for AI/ML Systems - 3", "requirementText": "The entity should classify all AI/ML assets according to factors such as their criticality to business operations, associated risk, and sensitivity and assign appropriate cyber security controls based on this classification." }, { "requirementID": "3.2.5 Application Security - 6", "requirementText": "The entity should maintain an up-to-date inventory of AI/ML applications, including their dependencies, and monitor them for any changes that could introduce cyber security risks." } ] } }, "Databricks": { "The Databricks AI Security Framework": { "link": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirements": [ { "requirementID": "DASF 16: Secure model features\n", "requirementText": "Databricks Feature Store is a centralized repository that enables data scientists to find and share features and also ensures that the same code used to compute the feature values is used for model training and inference. Unity Catalog’s capabilities, such as security, lineage, table history, tagging and cross-workspace access, are automatically available to the feature table to reduce the risk of malicious actors manipulating the features that feed into ML training." }, { "requirementID": "DASF 52: Source code control\n", "requirementText": "Databricks’ Git Repository integration supports effective code and third-party libraries management, enhancing customer control over their development environment." }, { "requirementID": "DASF 53: Third-party library control\n", "requirementText": "Databricks’ library management system allows administrators to manage the installation and usage of third-party libraries effectively. This feature enhances the security and efficiency of systems, pipelines and data by giving administrators precise control over their development environment." } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.2.3-1", "requirementText": "Developers and System Operators shall follow secure software supply chain processes for their AI model and system development." }, { "requirementID": "Provision 5.2.3-2", "requirementText": "System Operators that choose to use or adapt any models, or components, which are not well-documented or secured shall be able to justify their decision to use such models or components through documentation (for example if there was no other supplier for said component)." }, { "requirementID": "Provision 5.2.3-2.1", "requirementText": "In this case, Developers and System Operators shall have mitigating controls and undertake a risk assessment linked to such models or components." }, { "requirementID": "Provision 5.2.3-2.2", "requirementText": "System Operators shall share this documentation with End-users in an accessible way." }, { "requirementID": "Provision 5.2.3-3", "requirementText": "Developers and System Operators shall re-run evaluations on released models that they intend on using." }, { "requirementID": "Provision 5.2.3-4", "requirementText": "System Operators shall communicate their intention to update models to End-users in an accessible way prior to models being updated." } ] }, "SAI 002 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_gr/SAI/001_099/002/01.01.01_60/gr_SAI002v010101p.pdf", "requirements": [ { "requirementID": "6.1.3 Supply chain security - 2", "requirementText": "Setting minimum security standards for the supply chain and communicating these to the suppliers." }, { "requirementID": "6.1.3 Supply chain security - 4", "requirementText": "Adopting a view of supply chain security as a continuous process." }, { "requirementID": "6.5 - Fine-tuning", "requirementText": "And/or regular retraining of models with locally-verified or otherwise trusted data, where possible." } ] }, "TR 104 048 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_tr/104000_104099/104048/01.01.01_60/tr_104048v010101p.pdf", "requirements": [ { "requirementID": "6.1.3 Supply chain security - 1", "requirementText": "Understanding the risks associated with the supply chain, particularly for high-value components such as datasets. This includes understanding the security posture of the suppliers." }, { "requirementID": "6.1.3 Supply chain security - 2", "requirementText": "Setting minimum security standards for the supply chain and communicating these to the suppliers." }, { "requirementID": "6.1.3 Supply chain security - 3", "requirementText": "Building data and model security considerations into the contracting processes." }, { "requirementID": "6.1.3 Supply chain security - 4", "requirementText": "Adopting a view of supply chain security as a continuous process." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.2.3-1", "requirementText": "\"Developers and System Operators shall follow secure software supply chain processes for their AI model and system development.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout secure software supply chain processes, AI systems are vulnerable to risks like supply chain attacks, insertion of malicious components, and dependency issues, including models and datasets, which can compromise AI integrity and security. Lack of accountability across the supply-chain for jurisdiction-specific regulations can lead to data breaches and lack of regulatory compliance.\n\nExample Measures/Controls 1:\nSafeguard Provenance and Transparency: Mandate in internal standards that components can only be sourced by trusted and approved sources, documenting source, version, licencing, history, and other related artifacts (e.g. Model Card for models); use checksums to verify integrity. SBOMs can help automated creation of signed attestations of all components and their metadata. This can help safeguard transparency and provenance with non-repudiation and component tampering checks. SBOMs cover libraries and system components but not models or datasets. AI and ML BOMs are an emerging field. Monitor progress in standards such as Cyclone DX ML BOM [i.106] an introduce similar scans when tools emerge. In the meanwhile, rely on ML Ops to enforce model and dataset provenance and use file checksum to verify integrity and provenance.\n\nExample Measures/Controls 2:\nApply Vulnerability Management: Implement a vulnerability management process that includes regular (e.g. daily) scanning of third-party components for known vulnerabilities and timely patching to mitigate risks.\n\nExample Measures/Controls 3:\nAdopt Secure Supply Chain Frameworks: Follow the organization's preferred secure supply chain guidance or standard for all stages of AI development, from sourcing and integrating components to testing and deployment." }, { "requirementID": "Provision 5.2.3-2", "requirementText": "\"System Operators that choose to use or adapt any models, or components, which are not well-documented or secured shall be able to justify their decision to use such models or components through documentation (for example if there was no other supplier for said component).\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nUtilizing poorly documented or untrusted AI components because of some of their unique features introduces potential vulnerabilities, increasing risks of data leaks or unexpected behaviour.\n\nExample Measures/Controls:\nDocument Justification for Untrusted Components: When using poorly documented or untrusted components, document the justification, including alternative evaluations, and the absence of better suppliers." }, { "requirementID": "Provision 5.2.3-2.1", "requirementText": "\"In this case, Developers and System Operators shall have mitigating controls and undertake a risk assessment linked to such models or components.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nUtilizing poorly documented or untrusted AI components because of some of their unique features introduces potential vulnerabilities, increasing risks of data leaks or unexpected behaviour.\n\nExample Measures/Controls:\nEvaluate and Mitigate Risks for Untrusted Components: Perform a risk assessment to identify potential risks associated with unsecured AI components, specifying, and implementing mitigating controls to minimize vulnerabilities." }, { "requirementID": "Provision 5.2.3-2.2", "requirementText": "\"System Operators shall share this documentation with End-users in an accessible way.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nUtilizing poorly documented or untrusted AI components because of some of their unique features introduces potential vulnerabilities, increasing risks of data leaks or unexpected behaviour.\n\nExample Measures/Controls:\nShare Documentation with End-Users: Share documentation of non-compliant components with end-users, detailing risks, and justifications for transparency." }, { "requirementID": "Provision 5.2.3-3", "requirementText": "\"Developers and System Operators shall re-run evaluations on released models that they intend on using.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout reusable evaluations, periodic re-evaluation can be difficult, resulting into performance degradation, bias, inaccuracies, or security vulnerabilities go unnoticed in new releases.\n\nExample Measures/Controls:\nCreate and Maintain Appropriate and Reusable Model Evaluation Suites: Create appropriate model evaluation suites to assess performance, accuracy, security, and potential drift when required." }, { "requirementID": "Provision 5.2.3-4", "requirementText": "\"System Operators shall communicate the intention to update models to End-users in an accessible way prior to models being updated.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nLack of transparency around model updates can make it challenging for users to use a new model version, especially in evaluating data protection compliance. This creates availability and operational issues for end-users relying on certain model feature or behaviour.\n\nExample Measures/Controls:\nProvide Advance Notice of Model Updates: Notify end-users of model updates at least one month in advance, including any potential impacts on model performance or functionality and offer previews to test changes for at least a month. Ensure notices are accessible." } ] }, "TR 104 222 - Securing Artificial Intelligence; Mitigation Strategy Report": { "link": "https://www.etsi.org/deliver/etsi_tr/104200_104299/104222/01.02.01_60/tr_104222v010201p.pdf", "requirements": [ { "requirementID": "5.2.2 - Enhance data quality - 1", "requirementText": "One mitigation is to protect the data supply chain against manipulations to thwart attacks which tamper with or degrade the quality of training data. The risk for incorrect or manipulated data is much lower if data is sampled from a controlled environment." } ] } }, "Federal Office for Information Security": { "AI Security Concerns in a Nutshell": { "link": "https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/KI/Practical_Al-Security_Guide_2023.pdf?__blob=publicationFile&v=5", "requirements": [ { "requirementID": "5.3 Defending against Poisoning and Backdoor Attacks - Use Trusted Sources", "requirementText": "Depending on the security requirements for the use case, it is essential to make adequate efforts to ensure that the supply chain and the sources of training data, models, and code are known and trustworthy. Publicly available models could contain backdoors." }, { "requirementID": "5.3 Defending against Poisoning and Backdoor Attacks - Use Trusted Sources", "requirementText": "Retraining a model with benign training samples, if available, reduces the probability of backdoors being successful [18]. The degree of success depends on the size and quality of the clean dataset [22]. Research suggests that even with a small retraining dataset, the vulnerability of a model to backdoor attacks significantly drops, while its accuracy may be slightly reduced [22]." }, { "requirementID": "5.3 Defending against Poisoning and Backdoor Attacks - Network Pruning", "requirementText": "An autoencoder is trained with a benign dataset whose feature distribution is close to the training dataset. As a result, the trained autoencoder may be able to detect manipulated data samples that lie outside of the learned distribution [22]." } ] } }, "Google": { "Secure AI Framework": { "link": "https://www.saif.google/secure-ai-framework", "requirements": [ { "requirementID": "Model and Data Integrity Management", "requirementText": "Ensure that all data, models, and code used to produce AI models are verifiably integrity-protected during development and deployment." }, { "requirementID": "Secure-by-Default ML Tooling", "requirementText": "Use secure-by-default frameworks, libraries, software systems, and hardware components for AI development or deployment to protect confidentiality and integrity of AI assets and outputs." } ] } }, "ICO": { "Guidance on the AI Auditing Framework - Draft guidance for consultation ": { "link": "https://ico.org.uk/media2/about-the-ico/consultations/2617219/guidance-on-the-ai-auditing-framework-draft-for-consultation.pdf", "requirements": [ { "requirementID": "Outsourcing and third-party AI systems", "requirementText": "When you either buy an AI solution from a third party, or outsource it altogether, you need to conduct an independent evaluation of any trade-offs as part of your due diligence process. You are also required to specify your requirements at the procurement stage, rather than addressing trade-offs ex post." }, { "requirementID": "Preventative Controls - 6", "requirementText": "Document contracts with third parties are clear about the role and responsibilities of third parties." }, { "requirementID": "Preventative Controls - 7", "requirementText": "Document policies / processes for dealing with third parties and evidence of the due diligence of information security completed." } ] } }, "MIC/METI (Japan)": { "AI Guidelines for Business": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirements": [ { "requirementID": "Safety - 3 (b)", "requirementText": "Properly take actions such as the securement of transparency of data used for training, compliance with the legal framework, and update of AI models, within reasonable extent." } ] } }, "Microsoft": { "Responsible AI Standard": { "link": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirements": [ { "requirementID": "A4.3", "requirementText": "If you plan to use existing data sets to train the system, assess the quantity and suitability of available data sets that will be needed by the system in relation to the data requirements defined in A4.1. Document this assessment in the Impact Assessment." }, { "requirementID": "A4.5", "requirementText": "Evaluate all data sets using the methods defined in requirement A4.4. Document the results of the evaluation." } ] } }, "MITRE": { "ATLAS Framework": { "link": "https://atlas.mitre.org/mitigations", "requirements": [ { "requirementID": "AML.M0023 - AI Bill of Materials", "requirementText": "An AI Bill of Materials (AI BOM) contains a full listing of artifacts and resources that were used in building the AI. The AI BOM can help mitigate supply chain risks and enable rapid response to reported vulnerabilities.\n\nThis can include maintaining dataset provenance, i.e. a detailed history of datasets used for AI applications. The history can include information about the dataset source as well as well as a complete record of any modifications." } ] }, "SAFE-AI": { "link": "https://atlas.mitre.org/pdf-files/SAFEAI_Full_Report.pdf", "requirements": [ { "requirementID": "Supply chain and life cycle infiltrations and unvetted changes to the model (especially open source)", "requirementText": "AI-enabled systems often incorporate pre-trained models obtained from external sources. Failure to assure that these models are securely sourced from external suppliers will enable attacks that insert malicious code into the system that can compromise the AI system’s security and integrity. It is also important to scrutinize any updates or changes to these models, since the impact of a revised model on system behavior may not be immediately obvious. This makes change management and configuration management critically important for models. Continual/continuous testing may be needed to detect unexpected changes over time. It may also be important to establish baselines and understanding of operational data and its drift." }, { "requirementID": "Supply chain and life cycle infiltrations and unvetted changes of training and operational data", "requirementText": "AI-enabled systems often rely on data obtained from external sources. Failure to assure that these data resources are securely sourced from external suppliers will enable attacks that insert malicious code into the system that can compromise the AI system’s security and integrity. If the provenance of the data from external sources is not carefully documented (e.g., origins, transformations, dependencies, metadata, etc.), it may be challenging to recognize changes that render the data unsuitable for the current system. Mitigations include stringent vetting of AI model training, use of operational data, and thorough documentation of the data provenance." }, { "requirementID": "Supply chain infiltrations/ unvetted changes of AI tools/platform s (especially open source)", "requirementText": "AI-enabled systems are often built using tools and platforms obtained from external sources. If one of these resources has vulnerabilities because it is outdated, unpatched or compromised, it could provide a point of entry for attacks that poison data, perturb model inputs, or modify AI models to undermine their reliability, integrity and availability. Similar concerns arise if the external resources are not securely sourced from external suppliers. The vulnerabilities of the tools and platforms from external sources must be carefully understood and managed. Otherwise, it may be challenging to identify the underlying cause of any adverse outcomes in the AI system. Mitigations include meticulous attention to the preparation and maintenance of relevant risk assessment documents such as software bills of materials (SBOMs), AI system bills of materials (AIBOMs), data cards, and model cards." }, { "requirementID": "Supply Chain infiltration/ unvetted changes of Environment components (especially open source)", "requirementText": "Since AI is such a rapidly evolving technical area, AIenabled systems tend to incorporate open-source components, or capabilities provided by external suppliers. Adversaries can sometimes gain initial access to a system by infiltrating and compromising targeted portions of the AI supply chain. This could include specialized hardware like GPUs, software stacks for AI code development, or pretrained AI models. Failure to assure that AI-related components are securely sourced from external suppliers will enable attacks that insert malicious code into the system that can compromise the AI system’s security and integrity. Moreover, it is especially important to scrutinize all updates and patches needed for any third party or open-source capabilities that may be integrated into an AI component. This implies that relevant risk assessment documents must be carefully prepared, such as software bills of materials (SBOMs), AI system bills of materials (AIBOMs), data cards, and model cards." }, { "requirementID": "AI bias", "requirementText": "Biases in the data and models associated with an AI-enabled system can lead to inaccurate outcomes or discriminatory treatment of certain individuals or demographics, with a corresponding negative impact on the trustworthiness of the system. A key underlying issue is that the scale and complexity of many AI-enabled systems can result in high levels of statistical uncertainty and many potential sources of bias, making bias management a challenge. Unintended sources of bias like spurious correlations and unrepresentative data sources may be difficult to avoid. Malicious sources of bias caused by adversarial attacks on data and models may be challenging to detect. Mitigations include careful attention to the quality of data and its statistical properties, stringent access controls for data and models, and vigilant monitoring of system performance." }, { "requirementID": "Insecure Plugin Design", "requirementText": "AI software often extends its functionality by using plugins, extensions, or APIs to connect to other services or resources. Plugins may provide a variety of useful capabilities, such as integrations with other applications, access to public or private data sources, and the ability to execute code. If these plugins are not securely designed (e.g., plugins have insufficient access controls or inadequate input validation), adversaries may exploit their access to the AI software to compromise the plugins with attacks that have harmful consequences like data exfiltration, remote code execution, and privilege escalation. Developers must implement robust security measures for plugins, like strict parameterized inputs and secure access control guidelines, to mitigate this potential vulnerability." } ] } }, "Multi Agency": { "Guidelines for secure AI system development": { "link": "https://www.ncsc.gov.uk/files/Guidelines-for-secure-AI-system-development.pdf", "requirements": [ { "requirementID": "Secure your supply chain", "requirementText": "You assess and monitor the security of your AI supply chains across a system’s life cycle, and require suppliers to adhere to the same standards your own organisation applies to other software. If suppliers cannot adhere to your organisation’s standards, you act in accordance with your existing risk management policies.\n\nWhere not produced in-house, you acquire and maintain well-secured and well-documented hardware and software components (for example, models, data, software libraries, modules, middleware, frameworks, and external APIs) from verified commercial, open source, and other third-party developers to ensure robust security in your systems.\n\nYou are ready to failover to alternate solutions for mission-critical systems, if security criteria are not met. You use resources like the NCSC’s Supply Chain Guidance and frameworks such as Supply Chain Levels for Software Artifacts (SLSA) for tracking attestations of the supply chain and software development life cycles." } ] } }, "NCSC/NSA/CISA etc": { "AI Data Security\n": { "link": "https://media.defense.gov/2025/May/22/2003720601/-1/-1/0/CSI_AI_DATA_SECURITY.PDF", "requirements": [ { "requirementID": "2.1 Dataset Verification", "requirementText": "Before ingest, the consumer or curator should verify, as much as possible, that the dataset to be ingested is free of malicious or inaccurate material. Any detected abnormalities should be addressed, and suspicious data should not be stored. The dataset verification process should include a digital signature of the dataset at time of ingestion." }, { "requirementID": "2.2 Content Credentials", "requirementText": "Use content credentials to track the provenance of media and other data. Content credentials are “metadata that are secured cryptographically and allow creators the ability to add information about themselves or their creative process, or both, directly to media content…. Content Credentials securely bind essential metadata to a media file that can track its origin(s), any edits made, and/or what was used to create or modify the content…. This metadata alone does not allow a consumer to determine whether a piece of content is ‘true,’ but rather provides contextual information that assists in determining the authenticity of the\ncontent.” [24]" }, { "requirementID": "2.3 Foundation model assurances", "requirementText": "In the case where a consumer is not ingesting a dataset but a foundation model trained by another party, the developers of the foundation model need to be able to provide assurances regarding the data and sources used and certify that their training data did not contain any known compromised data. Take care to track the training data used in various model lineages. Exercise caution before using a model without such assurances." }, { "requirementID": "2.4 Require certification", "requirementText": "Data consumers should strongly consider requiring a formal certification from dataset and model providers, attesting that their systems are free from known compromised data before using third-party data and/or foundation models." }, { "requirementID": "2.10 Certification by Curators", "requirementText": "Since the data supply chain begins with the curators, the certification process must start there as well. To the best of their ability, curators should be able to certify that, at the time of publication, the dataset contains no malicious or inaccurate material." }, { "requirementID": "2.11 Test & verify web-scale datasets", "requirementText": "Be cautious when using web-scale datasets that are vulnerable to frontrunning poisoning. Check that the data hasn’t been manipulated, and only use snapshots verified by a trusted party." }, { "requirementID": "2.12 (For web-scale data collectors) Randomize or lengthen snapshots", "requirementText": "Collectors such as Wikipedia should defend against actors making malicious edits ahead of a planned snapshot by: 1. Randomizing the snapshot order. 2. Freezing edits to content long enough for edits to go through review before releasing the snapshot." }, { "requirementID": "2.13 Consensus approaches", "requirementText": "Data consumers using web-crawled datasets should rely on consensus-based approaches, since notional determinations of which domains to trust are ad-hoc and insufficient. For example, an AI developer could choose to only trust an image-caption pair when it appears on many different websites to reduce susceptibility to poisoning techniques, since a malicious actor would have to poison a sufficiently large number of websites to be successful." }, { "requirementID": "2.14 Data curation", "requirementText": "Ultimately, it is incumbent on organizations to ensure malicious or inaccurate material is not present in the data they use. If an organization does not have resources to conduct the necessary due diligence, then the use of web-crawled datasets is not recommended until some sort of trust infrastructure can be implemented" }, { "requirementID": "3.1 Anomaly Detection", "requirementText": "Incorporate anomaly detection algorithms during data preprocessing to identify and remove malicious or suspicious data points before training. These algorithms can recognize statistically deviant patterns in the data, making it possible to isolate and eliminate poisoned inputs" } ] } }, "NIST": { "AI 100-2e2025: Adversarial Machine Learning\nA Taxonomy and Terminology of Attacks and Mitigations": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-2e2025.pdf", "requirements": [ { "requirementID": "2.3.1 Training data sanitization", "requirementText": "These methods leverage the insight that poisoned samples are typically different than regular training samples that are not controlled by adversaries. As such, data sanitization techniques are designed to clean the training set and remove the poisoned samples before the ML training is performed. Cretu et al. [96] proposed the first sanitization procedure for unlabeled datasets that relies on majority voting of multiple models trained on subsets of the training set. They apply the method to anomaly detection on network packets. Nelson et al. [269] introduced the Region of Non-Interest (RONI) method, which examines each sample and excludes it from training if the accuracy of the model decreases when the sample is added. Subsequently proposed sanitization methods improved upon these early approaches by reducing their computational complexity and considering other applications. Paudice et al. [289] introduced a method for label cleaning that was specifically designed for label-flipping attacks. Steinhardt et al. [354] proposed the use of outlier detection methodsfor identifying poisoned samples. Clustering methods have also been used to detect poisoned samples[203, 363]. Other work hassuggested that computing the variance of predictions made by an ensemble of multiple ML models is an effective data sanitization method for network intrusion detection [384]. Once sanitized, datasets may be protected by cybersecurity mechanisms for provenance and integrity attestation [267]" }, { "requirementID": "2.3.3 Training data sanitization", "requirementText": "Similar to poisoning availability attacks, training data sanitization can be applied to detecting backdoor poisoning attacks. For example, outlier detection in the latent feature space [157, 293, 378] has been effective for convolutional neural networks used for computer vision applications. Activation Clustering [76] clusters training data in representation space to isolate the backdoored samplesin a separate cluster. Data sanitization achieves better results when the poisoning attack controls a relatively large fraction of training data but is not as effective against stealthy poisoning attacks. Overall, this leads to a trade-off between attack success and the detectability of malicious samples." }, { "requirementID": "2.3.4 Model Poisoning", "requirementText": "A variety of Byzantine-resilient aggregation rules have been designed and evaluated to defend federated learning from model poisoning attacks. Most of them attempt to identify and exclude the malicious updates when performing the aggregation at the server [8, 43, 51, 149, 242–244, 359, 423]. However, motivated adversaries can bypass these defenses by adding constraints to the attack generation optimization problem [23, 123, 335]. Gradient clipping and differential privacy have the potential to mitigate model poisoning attacksto some extent [23, 271, 360], but they usually decrease accuracy and do not provide complete mitigation.\nFor specific model poisoning vulnerabilities,such as backdoor attacks, there are some techniques for model inspection and sanitization (see Sec. 2.3.3). However, mitigating supplychain attacks in which adversaries might control the source code of the training algorithm or the ML hyperparameters remains challenging. Program verification techniques used in other domains(e.g., cryptographic protocol verification [299])might be adapted to thissetting, but ML algorithms have intrinsic randomness and non-deterministic behavior, which enhances the difficulty of verification." }, { "requirementID": "3.3.3 Direct Prompting Attacks - Training data sanitization", "requirementText": "Model training data can be sanitized to remove sensitive or toxic content and data that are largely or exclusively relevant for developing undesirable capabilities. Such sanitization may prevent harmful capabilities from being learned and reduce the potential harms from direct prompt injection, though they may harm generalization and harmful content detection abilities [224]." } ] }, "AI RMF 1.0": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirements": [ { "requirementID": "GOVERN 6.1", "requirementText": "Policies and procedures are in place that address AI risks associated with third-party entities, including risks of infringement of a third-party’s intellectual property or other rights." }, { "requirementID": "GOVERN 6.2", "requirementText": "Contingency processes are in place to handle failures or incidents in third-party data or AI systems deemed to be high-risk." }, { "requirementID": "MAP 4.1", "requirementText": "Approaches for mapping AI technology and legal risks of its components – including the use of third-party data or software – are in place, followed, and documented, as are risks of infringement of a third party’s intellectual property or other rights." } ] }, "IR 8596: Cybersecurity Framework Profile for Artificial Intelligence (Cyber AI Profile): NIST Community Profile": { "link": "https://csrc.nist.gov/pubs/ir/8596/iprd", "requirements": [ { "requirementID": "GV.SC-03", "requirementText": "Cybersecurity supply chain risk management is integrated into cybersecurity and enterprise risk management, risk assessment, and improvement processes" }, { "requirementID": "GV.SC-04", "requirementText": "Suppliers are known and prioritized by criticality" }, { "requirementID": "GV.SC-05", "requirementText": "Requirements to address cybersecurity risks in supply chains are established, prioritized, and integrated into contracts and other types of agreements with suppliers and other relevant third parties" }, { "requirementID": "GV.SC-06", "requirementText": "Planning and due diligence are performed to reduce risks before entering into formal supplier or other third-party relationships" }, { "requirementID": "GV.SC-07", "requirementText": "The risks posed by a supplier, their products and services, and other third parties are understood, recorded, prioritized, assessed, responded to, and monitored over the course of the relationship" }, { "requirementID": "GV.SC-08", "requirementText": "Relevant suppliers and other third parties are included in incident planning, response, and recovery activities" }, { "requirementID": "GV.SC-09", "requirementText": "Supply chain security practices are integrated into cybersecurity and enterprise risk management programs, and their performance is monitored throughout the technology product and service life cycle" }, { "requirementID": "GV.SC-10", "requirementText": "Cybersecurity supply chain risk management plans include provisions for activities that occur after the conclusion of a partnership or service agreement" }, { "requirementID": "ID.AM-04", "requirementText": "Inventories of services provided by suppliers are maintained" }, { "requirementID": "ID.RA-09", "requirementText": "The authenticity and integrity of hardware and software are assessed prior to acquisition and use" }, { "requirementID": "ID.RA-10", "requirementText": "Critical suppliers are assessed prior to acquisition" } ] }, "SP 800-218A": { "link": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirements": [ { "requirementID": "PO.3.1", "requirementText": "Specify which tools or tool types must or should be included in each toolchain to mitigate identified risks, as well as how the toolchain components are to be integrated with each other.\n\nPlan to develop and implement automated toolchains that secure AI model development and reduce human effort, especially at the scale often used by AI models." }, { "requirementID": "PS.3.2", "requirementText": "Collect, safeguard, maintain, and share provenance data for all components of each software release (e.g., in a software bill of materials [SBOM], through Supply-chain Levels for Software Artifacts [SLSA]).\n\nTrack the provenance of an AI model and its components and derivatives, including the training libraries, frameworks, and pipelines used to build the model.\n\nTrack AI models that were trained on sensitive data (e.g., payment card data, protected health information, other types of personally identifiable information), and determine if access to the models should be restricted to individuals who already have access to the sensitive data used for training." }, { "requirementID": "PW.4.4", "requirementText": "Verify that acquired commercial, open-source, and all other third-party software components comply with the requirements, as defined by the organization, throughout their life cycles.\n\nVerify the integrity, provenance, and security of an existing AI model or any other acquired AI components — including training, testing, fine-tuning, and aligning datasets; reward models; adaptation layers; and configuration parameters — before using them. \n\nScan and thoroughly test acquired AI models and their components for vulnerabilities and malicious content before use." } ] } }, "OWASP": { "LLM Top 10": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirements": [ { "requirementID": "LLM03: Supply Chain - 4", "requirementText": "Maintain an up-to-date inventory of components using a Software Bill of Materials (SBOM) to ensure you have an up-to-date, accurate, and signed inventory, preventing tampering with deployed packages. SBOMs can be used to detect and alert for new, zero-date vulnerabilities quickly. AI BOMs and ML SBOMs are an emerging area and you should evaluate options starting with OWASP CycloneDX" }, { "requirementID": "LLM03: Supply Chain - 5", "requirementText": " To mitigate AI licensing risks, create an inventory of all types of licenses involved using BOMs and conduct regular audits of all software, tools, and datasets, ensuring compliance and transparency through BOMs. Use automated license management tools for real-time monitoring and train teams on licensing models. Maintain detailed licensing documentation\nin BOMs." }, { "requirementID": "LLM03: Supply Chain - 6", "requirementText": "Only use models from verifiable sources and use third-party model integrity checks with signing and file hashes to compensate for the lack of strong model provenance. Similarly, use code signing for externally supplied code." }, { "requirementID": "LLM03: Supply Chain - 8", "requirementText": " AAnomaly detection and adversarial robustness tests on supplied models and data can help detect tampering and poisoning as discussed in \"LLM04 Data and Model Poisoning; ideally, this should be part of MLOps and LLM pipelines; however, these are emerging techniques and may be easier to implement as part of red teaming exercises." }, { "requirementID": "LLM04: Data and Model Poisoning - 1", "requirementText": "Track data origins and transformations using tools like OWASP CycloneDX or ML-BOM. Verify data legitimacy during all model development stages." }, { "requirementID": "LLM04: Data and Model Poisoning - 2", "requirementText": "Vet data vendors rigorously, and validate model outputs against trusted sources to detect signs of poisoning." } ] }, "OWASP Model Context Protocol (MCP) Top 10": { "link": "https://owasp.org/www-project-mcp-top-10/", "requirements": [ { "requirementID": "MCP04:2025 – Software Supply Chain Attacks & Dependency Tampering - 2", "requirementText": "Build SBOM / CBOM Visibility Generate SBOM (software bill of materials) and CBOM (cryptographic bill of materials) snapshots for each MCP server + plugin package Store SBOM alongside deployments for auditing + incident response" }, { "requirementID": "MCP04:2025 – Software Supply Chain Attacks & Dependency Tampering - 5", "requirementText": "Dependency Scanning\n- Apply SCA (software composition analysis) + code scanning tools to detect:\n- Known CVEs\n- Malicious indicators\n- Poisoned transitive dependencies" }, { "requirementID": "MCP04:2025 – Software Supply Chain Attacks & Dependency Tampering - 7", "requirementText": "Supply-Chain Governance\n- Maintain vendor risk profiles\n- Require suppliers to provide signed attestations\n- Review open-source maintainers’ security maturity" } ] }, "OWASP Top 10 for Agentic Applications for 2026": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "requirements": [ { "requirementID": "ASI04: Agentic Supply Chain Vulnerabilities - 1", "requirementText": "Provenance and SBOMs, AIBOMs: Sign and attest manifests, prompts, and tool definitions; require and operationalize SBOMs, AIBOMs with periodic attestations; maintain inventory of AI components; use curated registries and block untrusted sources." }, { "requirementID": "ASI04: Agentic Supply Chain Vulnerabilities - 2", "requirementText": "Dependency gatekeeping: Allowlist and pin; scan for typosquats (PyPI, npm, LangChain, LlamaIndex); verify provenance before install or activation; auto-reject unsigned or unverified." }, { "requirementID": "ASI06: Memory & Context Poisoning - 5", "requirementText": "Provenance and anomalies: Require source attribution and detect suspicious updates or frequencies." }, { "requirementID": "ASI07: Insecure Inter-Agent Communication - 8", "requirementText": "Attested registry and agent verification: Use registries or marketplaces that provide digital attestation of agent identity, provenance, and descriptor integrity. Require signed agent cards and continuous verification before accepting discovery or coordination messages. Leverage the PKI trusted root certificate registries to enable robust agent verification and attestation of critical attributes." } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirements": [ { "requirementID": "10.5", "requirementText": "Al Systems must highlight the Provider." }, { "requirementID": "12.1", "requirementText": "An Entity that outsources any activity to support its operations when developing, providing, using, or implementing an Al System should ensure regular due diligence on the outsourcing service provider is carried out (identity, legal status, activities, financial position, etc.) and obtain prior consent from QCB." } ] } }, "SANS": { "Critical AI Security Guidelines": { "link": "https://sansorg.egnyte.com/dl/bvkYQxrW8QMj", "requirements": [ { "requirementID": "3.4 Be Cautious Using Public Models", "requirementText": "Some of the mechanisms used to share models can be leveraged by bad actors to introduce malicious code into the packaging used to deploy the model. In other words, the model itself has not been tampered with, but the package the model is inside of has been built with malicious actions that will be executed when the model is unpacked or called. Models also may be created by bad actors with architectural backdoors in them. The idea of a backdoor like this would be to invoke a specific behavior in response to a specific input. Once a backdoor is created inside of a model, it can be difficult, if not impossible, to remove it via fine-tuning. This becomes an issue if a user inadvertently triggers the backdoor or, if it is exposed outside the organization, a bad actor attempts to exploit it across a wide range of models." }, { "requirementID": "6.3 Maintain an AI Bill of Materials", "requirementText": "LLM applications depend upon a complex underlying ecosystem for their functionality. Modeled after software bill of materials (SBOM), creation and maintenance of an AIBOM can provide better visibility into relevant aspects of the AI supply chain, including considerations of dataset and model provenance. AIBOMs contain technical details that are useful to adversaries in attacking LLM applications. Care should be taken to limit the disclosure of AIBOMs." } ] } }, "Smart Dubai (UAE)": { "AI Ethics Principles & Guidelines": { "link": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirements": [ { "requirementID": "1.2.7.1", "requirementText": "In the case of critical decisions, AI operator organisations should avoid using AI systems that cannot be subjected to meaningful standards of accountability and transparency." } ] } }, "TAIBOM": { "Bringing Trustworthiness to AI-Enabled Systems": { "link": "https://arxiv.org/pdf/2510.02169", "requirements": [ { "requirementID": "UC 4: Evaluating CVE Impact on Training and Inference Systems", "requirementText": "New CVEs may affect software libraries used in AI training or inference. Identifying affected models requires understanding which code components were used and how they relate to model artifacts." } ] } } }, "Principle 8": { "CEN/CENELEC": { "prEN 40000-1-2: Cybersecurity requirements for products with digital elements - Part 1-2: Principles for cyber resilience": { "link": "https://genorma.com/en/standards/pren-40000-1-2", "requirements": [ { "requirementID": "5.5", "requirementText": "Transparency" } ] } }, "Central Bank of the UAE": { "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E": { "link": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirements": [ { "requirementID": "4. Transparency and Explain ability - b", "requirementText": "Institutions should maintain documentation relating to the design, training data, model assumptions and functioning of AI to facilitate internal review and internal and external audit." } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirements": [ { "requirementID": "2.2.6 - Data Quality and Availability", "requirementText": "Apply specialized domain knowledge that curates high-quality, comprehensive datasets for effective AI performance. This can be challenging in industrial settings; potential difficulties include the environment containing proprietary or outdated systems and bespoke solutions, and the difficulty of capturing safety-related edge cases. OT operator expertise is necessary for capturing this specialized knowledge. Additionally, operators should work with AI model developers to ensure a data integrity program covers AI systems." } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "A&A-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain audit and assurance policies and procedures and standards. Review and update the policies and procedures at least annually or upon significant changes." }, { "requirementID": "DSP-05", "requirementText": "Create data flow documentation to identify what data is processed, stored or transmitted where. Review data flow documentation at defined intervals, at least annually, and after any change." }, { "requirementID": "DSP-19", "requirementText": "Define and implement, processes, procedures and technical measures to specify and document the physical locations of data, including any locations in which data is processed or backed up." }, { "requirementID": "DSP-20", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to: 1) Document and trace data sources, and 2) Make the data source available according to legal and regulatory requirements" }, { "requirementID": "GRC-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for an information governance program, which is sponsored by the leadership of the organization and related to AI systems as well. Review and update the policies and procedures at least annually." }, { "requirementID": "GRC-13", "requirementText": "Establish, document, and communicate the degree of explainability needed for the AI Services." }, { "requirementID": "LOG-08", "requirementText": "Generate audit records containing relevant security information." }, { "requirementID": "MDS-03", "requirementText": "Define, implement, enforce, approve, document, communicate, maintain and evaluate processes and procedures for model documentation. Regularly review and update the model documentation." }, { "requirementID": "MDS-04", "requirementText": "Establish and implement baseline requirements for Model documentation." }, { "requirementID": "MDS-05", "requirementText": "Define, implement, and evaluate processes, procedures, and technical measures for the validation of the Model documentation aligned with the current model." }, { "requirementID": "MDS-08", "requirementText": "Regularly calculate and compare checksums using cryptographic hashes of model checkpoints to detect unauthorized modifications. Apply at least annually based on the level of risk, or after any change of hands." }, { "requirementID": "MDS-09", "requirementText": "Sign models cryptographically and verify signatures to ensure model provenance and ownership, any time the model changes hands or is loaded from storage." }, { "requirementID": "UEM-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for all endpoints. Review and update the policies and procedures at least annually or upon significant system changes." }, { "requirementID": "UEM-02", "requirementText": "Define, document, apply and evaluate a list of approved services, applications and sources of applications (stores) acceptable for use by endpoints when accessing or storing organization-managed data." } ] } }, "CoSAI": { "AI Incident Response Framework": { "link": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirements": [ { "requirementID": "3.3.2. Detection and Analysis Phase - Investigation Procedures - Evidence Collection", "requirementText": "• Interaction logs\n• System configurations\n• Model versions and parameters\n• Network traffic and API calls\n• Vector database queries" } ] }, "Establish Risks and Controls for the AI Supply Chain": { "link": "https://github.com/cosai-oasis/ws1-supply-chain/blob/main/risks-and-controls-for-the-ai-supply-chain-v1.md", "requirements": [ { "requirementID": "3.1.1 Data Poisoning: Threats and Mitigations in AI Supply Chains - Inadequate Data", "requirementText": "Using irrelevant data for model fine-tuning:\nComprehensive data provenance with mixture weight documentation" }, { "requirementID": "3.1.1 Data Poisoning: Threats and Mitigations in AI Supply Chains - Invalid Transformations", "requirementText": "Data corruption during cleaning processes:\nComplete lineage tracking of data transformations" }, { "requirementID": "3.1.4 AI Infrastructure Supply Chain - Opaque Infrastructure Provenance", "requirementText": "Insufficient visibility into hosting infrastructure components and their security posture:\nRequest detailed infrastructure documentation, attestations, and third-party audit results" }, { "requirementID": "3.2.2 - Lack of Model Provenance or Incorrect Provenance", "requirementText": "The model's provenance information may be invalid, incomplete, or deliberately tampered with to conceal attacks. In some cases, provenance documentation may be entirely absent, creating significant security gaps:\nGenerate comprehensive model provenance using trusted build systems and ensure it exists in a tamper-proof format. Implement strict policies preventing the deployment of models lacking proper provenance documentation." } ] }, "Model Context Protocol (MCP) Security": { "link": "https://github.com/cosai-oasis/ws4-secure-design-agentic-systems/blob/main/model-context-protocol-security.md", "requirements": [ { "requirementID": "3.2.6 Cryptographic Verification of Resources", "requirementText": "Organizations developing MCP servers must provide cryptographic signatures and software bill of materials (SBOMs) for all server code to verify provenance. Organizations deploying MCP clients and servers should obtain and verify the contents and cryptographic signatures prior to deployment, and have policies restricting the approved sources and signing keys. TLS should be used to protect all data in transit. Remote attestation can further verify that servers are running expected code in a trusted environment. When supported, end-to-end cryptographic signatures can prove the authenticity of resources returned by MCP servers." } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "3.1.4 Change Management and Reporting - 1", "requirementText": "The entity should establish and maintain a change management process for AI/ML systems, ensuring any manual or human-initiated modifications, upgrades, or alterations to models, algorithms, training data, or system configurations are properly documented, reviewed, and approved." } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.2.4-1", "requirementText": "Developers shall document and maintain a clear audit trail of their system design and post-deployment maintenance plans. Developers should make the documentation available to the downstream System Operators and Data Custodians." }, { "requirementID": "Provision 5.2.4-1.1", "requirementText": "Developers should ensure that the document includes security-relevant information, such as the sources of training data (including fine-tuning data and human or other operational feedback), intended scope and limitations, guardrails, retention time, suggested review frequency and potential failure modes." }, { "requirementID": "Provision 5.2.4-1.2", "requirementText": "Developers shall release cryptographic hashes for model components that are made available to other stakeholders to allow them to verify the authenticity of the components." }, { "requirementID": "Provision 5.2.4-2", "requirementText": "Where training data has been sourced from publicly available sources, there is a risk that this data might have been poisoned. As discovery of poisoned data is likely to occur after training (if at all), Developers shall document how they obtained the public training data, where it came from and how that data is used in the model." }, { "requirementID": "Provision 5.2.4-2.1", "requirementText": "The documentation of training data should include at a minimum the source of the data, such as the URL of the scraped page, and the date/time the data was obtained. This will allow Developers to identify whether a reported data poisoning attack was in their data sets." }, { "requirementID": "Provision 5.2.4-3", "requirementText": "Developers should ensure that they have an audit log of changes to system prompts or other model configuration (including prompts) that affect the underlying working of the systems. Developers can make this available to any System Operators and End-Users that have access to the model." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.2.4-1", "requirementText": "\"Developers shall document and maintain a clear audit trail of their system design and post-deployment maintenance plans. Developers should make the documentation available to the downstream System Operators and Data Custodians.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout thorough documentation and audit trails, AI system operations can lack transparency, limiting traceability and increasing risks of security gaps, regulatory non-compliance, and operational inefficiencies.\n\nExample Measures/Controls:\nDevelop Comprehensive System Design and Maintenance Documentation: Document the system design and postdeployment maintenance plans with audit trail of design decisions, architectural diagrams, and maintenance schedules. Ensure the documentation is accessible to downstream System Operators and Data Custodians, highlighting responsibilities, version control, and any dependencies." }, { "requirementID": "Provision 5.2.4-1.1", "requirementText": "\"Developers should ensure that the document includes security-relevant information, such as the sources of training data (including fine-tuning data and human or other operational feedback), intended scope and limitations, guardrails, retention time, suggested review frequency and potential failure modes.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nLack of detailed security-relevant documentation can lead to unmitigated risks from unverified data sources, misuse of the AI system, and challenges in addressing failure modes.\n\nExample Measures/Controls:\nInclude Relevant Security - Information in System Documentation: Document AI systems including intended scope, limitations, known failure modes, prompts, guardrails training data sources, data retention policies, review schedules. Use Model Cards to capture transparent summaries of a model's capabilities, ethical considerations, performance metrics, and limitations, consider using ML Bills of Materials (ML BOMs) to document in machinereadable way the model and its dependencies, with component versions, and licensing." }, { "requirementID": "Provision 5.2.4-1.2", "requirementText": "\"Developers shall release cryptographic hashes for model components that are made available to other stakeholders to allow them to verify the authenticity of the components.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nAbsence of cryptographic hashes for model components increases the risk of tampering, unauthorized modifications, and integrity issues, compromising trust and security.\n\nExample Measures/Controls:\nInclude Cryptographic Hashes for Models: Release cryptographic hashes for all models made available to stakeholders, enabling verification of component authenticity." }, { "requirementID": "Provision 5.2.4-2", "requirementText": "\"Where training data has been sourced from publicly available sources, there is a risk that this data might have been poisoned. As discovery of poisoned data is likely to occur after training (if at all), Developers shall document how they obtained the public training data, where it came from and how that data is used in the model.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout comprehensive documentation of training data sources and collection timestamps, organizations can be unable to determine whether their AI models have been affected by data poisoning, fraud, or insider abuse and whether they are compliant to data protection legislation. If data poisoning attacks are revealed to have occurred on public websites, producers or users of models will only know if they are affected if they have robust documentation of what data the model was trained on.\n\nExample Measures/Controls:\nDocument the process of sourcing public training data: Detailing how data was collected, processed, and used in the model (e.g. pretraining, fine-tuning). Include poisoning mitigation measures such as data validation and anomaly detection. Maintain an audit trail to trace datasets if poisoning is suspected." }, { "requirementID": "Provision 5.2.4-2.1", "requirementText": "\"The documentation of training data should include at a minimum the source of the data, such as the URL of the scraped page, and the date/time the data was obtained. This will allow Developers to identify whether a reported data poisoning attack was in their data sets.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nFailure to record detailed metadata, such as data sources and timestamps, can hinder efforts to trace and respond to data poisoning incidents.\n\nExample Measures/Controls:\nDocument metadata for all publicly sourced training data: Include the exact source (e.g. URLs) and date/time of collection. Ensure this metadata is stored in an accessible format to facilitate traceability in case of reported data poisoning. " }, { "requirementID": "Provision 5.2.4-3", "requirementText": "\"Developers should ensure that they have an audit log of changes to system prompts or other model configuration (including prompts) that affect the underlying working of the systems. Developers can make this available to any System Operators and End-Users that have access to the model.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout an audit log for configuration changes, tracking and understanding modifications to prompts or model configurations become difficult, increasing risks of unintended system behaviour or accountability issues.\n\nExample Measures/Controls:\nMaintain an Audit Log of Prompt and Model Configuration Changes: Implement an audit log that captures all changes to system prompts and configuration settings, recording details such as change date, user ID, and a description of modifications. Provide access to System Operators and Data Custodians." } ] }, "TS 104 224 - Securing Artificial Intelligence (SAI); Explicability and transparency of AI processing": { "link": "https://www.etsi.org/deliver/etsi_ts/104200_104299/104224/01.01.01_60/ts_104224v010101p.pdf", "requirements": [ { "requirementID": "5.2 (a)", "requirementText": "The statement of system purpose is critical in allowing a layperson to clearly understand the intent of the system and the role of AI in achieving that purpose or intent. " }, { "requirementID": "5.2 (b)", "requirementText": "The statement of system purpose should be written in natural language and be concise as well as precise (i.e. not open to variations in interpretation).\nThe following characteristics shall be identifiable in the statement of system purpose:\n• Unambiguous: it should be impossible to interpret the system purpose in more than one way.\n• Complete: the system purpose should contain all the information necessary to understand it without requiring reference to other documents.\n• Precise: the system purpose should be worded clearly and exactly, without unnecessary detail that might confuse the reader.\n• Well-structured: any individual elements of the system purpose should be included in an appropriate and easy-to-read manner." }, { "requirementID": "6.4.1", "requirementText": "The system documentation shall clearly identify those events that are logged for future analysis. The content of the log record shall be sufficient to identify the trigger conditions of the event and should include the following:\n• The time that the event occurred (including the metric and basis of the time).\nEXAMPLE: Time may be system clock time, e.g. UTC time, or may be relative to some datum (e.g. clock cycles from a known datum point).\n• The software versions of the processes involved or impacted in the event.\n• The hardware elements involved or impacted in the event.\n• The data, and its supply chain, involved or impacted in the event.\nIn addition the liable party for resolving the impact of the event should be clearly identified.\nNOTE: In the AI Act [i.5] it is stated that high risk systems have the capability to allow for automatic recording of events over the lifetime of the AI System and this is consistent with the requirements outlined in the present document. " } ] } }, "EU ": { "EU AI Act": { "link": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirements": [ { "requirementID": "11.1 Technical Documentation", "requirementText": "The technical documentation of a high-risk AI system shall be drawn up before that system is placed on the market or put into service and shall be kept up-to date. The technical documentation shall be drawn up in such a way as to demonstrate that the high-risk AI system complies with the requirements set out in this Section and to provide national competent authorities and notified bodies with the necessary information in a clear and comprehensive form to assess the compliance of the AI system with those requirements. It shall contain, at a minimum, the elements set out in Annex IV. SMEs, including start-ups, may provide the elements of the technical documentation specified in Annex IV in a simplified manner. To that end, the Commission shall establish a simplified technical documentation form targeted at the needs of small and microenterprises. Where an SME, including a start-up, opts to provide the information required in Annex IV in a simplified manner, it shall use the form referred to in this paragraph. Notified bodies shall accept the form for the purposes of the conformity assessment." }, { "requirementID": "11.2 Techincal Documentation", "requirementText": "Where a high-risk AI system related to a product covered by the Union harmonisation legislation listed in Section A of Annex I is placed on the market or put into service, a single set of technical documentation shall be drawn up containing all the information set out in paragraph 1, as well as the information required under those legal acts." }, { "requirementID": "11.3 Technical Documentation", "requirementText": "The Commission is empowered to adopt delegated acts in accordance with Article 97 in order to amend Annex IV, where necessary, to ensure that, in light of technical progress, the technical documentation provides all the information necessary to assess the compliance of the system with the requirements set out in this Section." }, { "requirementID": "18.1 Documentation Keeping", "requirementText": "1. The provider shall, for a period ending 10 years after the high-risk AI system has been placed on the market or put into service, keep at the disposal of the national competent authorities:\n(a) the technical documentation referred to in Article 11;\n(b) the documentation concerning the quality management system referred to in Article 17;\n(c) the documentation concerning the changes approved by notified bodies, where applicable;\n(d) the decisions and other documents issued by the notified bodies, where applicable;\n(e) the EU declaration of conformity referred to in Article 47." }, { "requirementID": "18.2 Documentation Keeping", "requirementText": "Each Member State shall determine conditions under which the documentation referred to in paragraph 1 remains at the disposal of the national competent authorities for the period indicated in that paragraph for the cases when a provider or its authorised representative established on its territory goes bankrupt or ceases its activity prior to the end of that period." }, { "requirementID": "18.3 Documentation Keeping", "requirementText": "Providers that are financial institutions subject to requirements regarding their internal governance, arrangements or processes under Union financial services law shall maintain the technical documentation as part of the documentation kept under the relevant Union financial services law." }, { "requirementID": "53.5 Obligations for providers of general purpose AI models", "requirementText": "For the purpose of facilitating compliance with Annex XI, in particular points 2 (d) and (e) thereof, the Commission is empowered to adopt delegated acts in accordance with Article 97 to detail measurement and calculation methodologies with a view to allowing for comparable and verifiable documentation." }, { "requirementID": "55.10 Obligations of Providors of General-Purpose AI models with Systemic Risk", "requirementText": "Any information or documentation obtained pursuant to this Article, including trade secrets, shall be treated in accordance with the confidentiality obligations set out in Article 78." } ] } }, "European Commission": { "Ethics guidelines for trustworthy AI": { "link": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirements": [ { "requirementID": "1.2.3 Accuracy", "requirementText": "Accuracy pertains to an AI system’s ability to make correct judgements, for example to correctly classify information into the proper categories, or its ability to make correct predictions, recommendations, or decisions based on data or models. An explicit and well-formed development and evaluation process can support, mitigate and correct unintended risks from inaccurate predictions. When occasional inaccurate predictions cannot be avoided, it is important that the system can indicate how likely these errors are. A high level of accuracy is especially crucial in situations where the AI system directly affects human lives." }, { "requirementID": "1.3.2 Quality and integrity of data", "requirementText": "The quality of the data sets used is paramount to the performance of AI systems. When data is gathered, it may contain socially constructed biases, inaccuracies, errors and mistakes. This needs to be addressed prior to training with any given data set. In addition, the integrity of the data must be ensured. Feeding malicious data into an AI system may change its behaviour, particularly with self-learning systems. Processes and data sets used must be tested and documented at each step such as planning, training, testing and deployment. This should also apply to AI systems that were not developed in-house but acquired elsewhere." }, { "requirementID": "1.4.1 Traceability", "requirementText": "The data sets and the processes that yield the AI system’s decision, including those of data gathering and data labelling as well as the algorithms used, should be documented to the best possible standard to allow for traceability and an increase in transparency. This also applies to the decisions made by the AI system. This enables identification of the reasons why an AI-decision was erroneous which, in turn, could help prevent future mistakes. Traceability facilitates auditability as well as explainability." }, { "requirementID": "2.1.3 Explanation Methods", "requirementText": "For a system to be trustworthy, we must be able to understand why it behaved a certain way and why it provided a given interpretation. A whole field of research, Explainable AI (XAI) tries to address this issue to better understand the system’s underlying mechanisms and find solutions. Today, this is still an open challenge for AI systems based on neural networks. Training processes with neural nets can result in network parameters set to numerical values that are difficult to correlate with results. Moreover, sometimes small changes in data values might result in dramatic changes in interpretation, leading the system to e.g. confuse a school bus with an ostrich. This vulnerability can also be exploited during attacks on the system. Methods involving XAI research are vital not only to explain the system’s behaviour to users, but also to deploy reliable technology." }, { "requirementID": "2.2.4 Certification", "requirementText": "As it cannot be expected that everyone is able to fully understand the workings and effects of AI systems, consideration can be given to organisations that can attest to the broader public that an AI system is transparent, accountable and fair. These certifications would apply standards developed for different application domains and AI techniques, appropriately aligned with the industrial and societal standards of different contexts. Certification can however never replace responsibility. It should hence be complemented by accountability frameworks, including disclaimers as well as review and redress mechanisms." } ] } }, "ICO": { "Guidance on the AI Auditing Framework - Draft guidance for consultation ": { "link": "https://ico.org.uk/media2/about-the-ico/consultations/2617219/guidance-on-the-ai-auditing-framework-draft-for-consultation.pdf", "requirements": [ { "requirementID": "Preventative Controls - 4", "requirementText": "Document policy / process for the separation of the AI development environment from the rest of the IT network / infrastructure. Evidence that the separation has been adhered to / happened." }, { "requirementID": "Preventative Controls - 9", "requirementText": "Adhere to the policy / process." }, { "requirementID": "Preventative Controls - 10", "requirementText": "Have a model governance policy." } ] } }, "ISO": { "42001:2023 - Information technology — Artificial intelligence — Management system": { "link": "https://www.iso.org/standard/42001", "requirements": [ { "requirementID": "7.5.2", "requirementText": "Creating and updating documented information" }, { "requirementID": "7.5.3", "requirementText": "Control of documented information" } ] } }, "ISO/IEC": { "TS 42119-2:2025": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:ts:42119:-2:ed-1:v1:en", "requirements": [ { "requirementID": "A.6", "requirementText": "Test documentation (summarized from ISO/IEC/IEEE 29119-3)" } ] } }, "MIC/METI (Japan)": { "AI Guidelines for Business": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirements": [ { "requirementID": "Accountability - 1", "requirementText": "Establish a situation that allows the origin of data and decisions made during the development, provision, or use of the AI system or service to be traced forward and backward to the extent that is reasonable and technically possible." }, { "requirementID": "Accountability - 6", "requirementText": "Document and store information on the items described above and keep them available for a prescribed period whenever and wherever required and able to be referenced in a manner appropriate for their use." } ] } }, "Microsoft": { "Responsible AI Standard": { "link": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirements": [ { "requirementID": "A4.4", "requirementText": "Define and document methods for evaluating data to be used by the system against the requirements defined in A4.1." }, { "requirementID": "A5.5", "requirementText": "Define and document Responsible Release Criteria to achieve this Goal. (Human oversight and control)" }, { "requirementID": "T1.1", "requirementText": "Identify:\n1) stakeholders who will use the outputs of the system to make decisions, and\n2) stakeholders who are subject to decisions informed by the system.\nDocument these stakeholders using the Impact Assessment template." }, { "requirementID": "T1.4", "requirementText": "Define and document a Responsible Release Plan, to include Responsible Release Criteria to achieve this Goal." }, { "requirementID": "T2.1", "requirementText": "Identify:\n1) stakeholders who make decisions about whether to employ a system for particular tasks, and\n2) stakeholders who develop or deploy systems that integrate with this system.\nDocument these stakeholders in the Impact Assessment template." }, { "requirementID": "T2.2", "requirementText": "Publish documentation for the system so that stakeholders defined in T2.1 can understand the system. Include:\n1) capabilities,\n2) intended uses,\n3) uses that require extra care or guidance,\n4) operational factors and settings that allow for effective and responsible system use,\n5) limitations, including uses for which the system was not designed or evaluated, and\n6) evidence of system accuracy and performance as well as a description of the extent to which these results are generalizable across use cases that were not part of the evaluation.\nWhen the system is a platform service made available to external customers or partners, a Transparency Note is required." }, { "requirementID": "T2.3", "requirementText": "Review and update documentation annually or when any of the following events occur:\n1) new uses are added,\n2) functionality changes,\n3) the product moves to a new release stage,\n4) new information about reliable and safe performance becomes known as defined by requirement RS3.3, or\n5) new information about system accuracy and performance becomes available.\nWhen the system is a platform service made available to external customers or partners, include this information in the required Transparency Note." }, { "requirementID": "F1.8", "requirementText": "Document the pre-release results from requirements F1.4, F1.5, and F1.6. Determine and document how often ongoing evaluation should be conducted to continue supporting this Goal." }, { "requirementID": "F2.7", "requirementText": "Identify and document any justifiable factors that account for any remaining differences between the rates at which resources and opportunities are allocated to identified demographic groups." }, { "requirementID": "F2.8", "requirementText": "Document the pre-release results for the evaluation described by requirements F2.4, F2.5, and F2.6. Determine and document how often ongoing evaluation should be conducted to continue supporting this goal." }, { "requirementID": "F3.6", "requirementText": "Document the pre-release results from requirements F3.4 and F3.5. Determine and document how often ongoing evaluation should be conducted to continue supporting this goal." }, { "requirementID": "RS1.1", "requirementText": "Document how:\n1) reliable and safe behavior is defined for this system and,\n2) what acceptable error rates are for overall system performance in the context of intended uses." }, { "requirementID": "RS1.3", "requirementText": "Determine and document the operational factors, including quality of system input, use, and operational context that are critical to manage for reliable and safe use of the system in its deployed context." }, { "requirementID": "RS1.4", "requirementText": "Define and document acceptable ranges for each operational factor important to support reliable and safe system use. Define and document an acceptable error rate for the system when operating within these ranges." }, { "requirementID": "RS1.9", "requirementText": "Provide documentation to customers and potential customers of the system that includes the outputs of requirements RS1.2, RS1.7 and RS1.8, and any unsupported uses defined in the Impact Assessment and in RS1.8. When the system is a platform service made available to external customers or partners, include this information in the required Transparency Note." }, { "requirementID": "RS3.1", "requirementText": "Establish and document a detailed inventory of the system health monitoring methods to be used, to include:\n1) data and insights generated from data repositories, system analytics, and associated alerts,\n2) processes by which customers can submit information about failures and concerns, and\n3) processes by which the general public can submit feedback." }, { "requirementID": "RS3.8", "requirementText": "Review and update documentation required by Goal T2 when any of the following events occur:\n1) new uses are added,\n2) functionality changes,\n3) new information about reliable and safe performance becomes known as defined by requirement RS3.3, or\n4) new information about system accuracy and performance becomes available.\nWhen the system is a platform service made available to external customers or partners, include this information in the required Transparency Note." } ] } }, "MITRE": { "ATLAS Framework": { "link": "https://atlas.mitre.org/mitigations", "requirements": [ { "requirementID": "AML.M0025 - Maintain AI Dataset Provenance", "requirementText": "Maintain a detailed history of datasets used for AI applications. The history should include information about the dataset's source as well as a complete record of any modifications." } ] } }, "Multi Agency": { "Guidelines for secure AI system development": { "link": "https://www.ncsc.gov.uk/files/Guidelines-for-secure-AI-system-development.pdf", "requirements": [ { "requirementID": "Document your data, models and prompts", "requirementText": "You document the creation, operation, and life cycle management of any models, datasets and meta- or system-prompts. Your documentation includes security-relevant information such as the sources of training data (including fine-tuning data and human or other operational feedback), intended scope and limitations, guardrails, cryptographic hashes or signatures, retention time, suggested review frequency and potential failure modes. Useful structures to help do this include model cards, data cards and software bills of materials (SBOMs). The production of comprehensive documentation supports transparency and accountability." } ] } }, "NCSC/NSA/CISA etc": { "AI Data Security\n": { "link": "https://media.defense.gov/2025/May/22/2003720601/-1/-1/0/CSI_AI_DATA_SECURITY.PDF", "requirements": [ { "requirementID": "1.1 Source Reliable Data and Track Data Provenance", "requirementText": "Verify data sources use trusted, reliable, and accurate data for training and operating AI systems. To the extent possible, only use data from authoritative sources. Implement provenance tracking to enable the tracing of data origins, and log the path that data follows through an AI system. [7] [8] [9] Incorporate a secure provenance database that is cryptographically signed and maintains an immutable, append-only ledger of data changes. This facilitates data provenance tracking, helps identify sources of maliciously modified data, and helps ensure that no single entity can undetectably manipulate the data." }, { "requirementID": "1.2 Verify and maintain data integrity during storage and transport", "requirementText": "Maintaining data integrity is an essential component to preserve the accuracy, reliability, and trustworthiness of AI data. [4] Use checksums and cryptographic hashes to verify that data has not been altered or tampered with during storage or transmission. Generating such unique codes for AI datasets enables the detection of unauthorized changes or corruption, safeguarding the information’s authenticity." }, { "requirementID": "1.3 Employ digital signatures to authenticate trusted data revisions", "requirementText": "Digital signatures help ensure data integrity and prevent tampering by third parties. Adopt quantum-resistant digital signature standards [5] [6] to authenticate and verify datasets used during AI model training, fine tuning, alignment, reinforcement learning from human feedback (RLHF), and/or other post-training processes that affect model parameters. Original versions of the data should be cryptographically signed, and any subsequent data revisions should be signed by the person who made the change. Organizations are encouraged to use trusted certificate authorities to verify this process." }, { "requirementID": "1.5 Classify data and use access controls", "requirementText": "Categorize data using a classification system based on sensitivity and required protection measures. [11] This process enables organizations to apply appropriate security controls to different data types. Classifying data enables the enforcement of robust protection measures like stringent encryption and access controls. [33] In general, the output of AI systems should be classified at the same level as the input data (rather than creating a separate set of guardrails)." }, { "requirementID": "2.6 Raw Data Hashes", "requirementText": "Data curators should attach a cryptographic hash to all raw data referenced in the dataset. This will enable follow-on data consumers to verify that the data has not changed since it was added to the list." }, { "requirementID": "2.7 Hash Verification", "requirementText": "Data consumers should incorporate a hash check at time of download in order to detect any changes made to it, and the downloader should discard any data that does not pass the hash check." }, { "requirementID": "3.6 Metadata Management", "requirementText": "Implement strong data governance practices to help ensure metadata is well-documented, complete, accurate, and secured." }, { "requirementID": "3.10 Representative Training Data", "requirementText": "Ensure that training data is representative of the totality of the information relevant to any given topic to reduce the risk of statistical bias. Also ensure that AI data is properly divided into training, development, and evaluation sets without overlap to properly measure statistical bias and other measures of performance." }, { "requirementID": "3.12 Test and Correct for Statiscial Bias", "requirementText": "Create a repository with instances of observed model output bias. Leverage that information to improve training data audits and with reinforcement learning to “undo” some of the measured bias." }, { "requirementID": "4.1 Data Management", "requirementText": "Employ a data management strategy in keeping with the best practices in this CSI to help ensure that it is easy to add and track new data elements for model training and adaptation. This management strategy enables identification of data elements causing drift for appropriate mitigation or action." } ] } }, "NIST": { "AI 800-1": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirements": [ { "requirementID": "Practice 7.1: Publish transparency reports - 1", "requirementText": "Share the methodology and results of pre- and post-deployment evaluations of model capabilities, risks, and mitigations, including as much detail about the data and evaluation methodology as can be disclosed without introducing risks to public safety." }, { "requirementID": "Practice 7.1: Publish transparency reports - 2", "requirementText": "Share details regarding the safeguards in place for the model, including how they are applied across different distribution channels, with as much detail as can be shared without rendering the safeguards ineffective." }, { "requirementID": "Practice 7.1: Publish transparency reports - 3", "requirementText": "Share information about data used to build the model that is relevant to assessing the misuse risk, such as criteria for data filtering, criteria for data selection, and data sources, when possible." } ] }, "AI RMF 1.0": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirements": [ { "requirementID": "MAP 2.1", "requirementText": "The specific tasks and methods used to implement the tasks that the AI system will support are defined (e.g., classifiers, generative models, recommenders)." }, { "requirementID": "MAP 2.2", "requirementText": "Information about the AI system’s knowledge limits and how system output may be utilized and overseen by humans is documented. Documentation provides sufficient information to assist relevant AI actors when making decisions and taking subsequent actions." }, { "requirementID": "MEASURE 2.9", "requirementText": "The AI model is explained, validated, and documented, and AI system output is interpreted within its context – as identified in the MAP function – to inform responsible use and governance." }, { "requirementID": "MEASURE 4.2", "requirementText": "Measurement results regarding AI system trustworthiness in deployment context(s) and across the AI lifecycle are informed by input from domain experts and relevant AI actors to validate whether the system is performing consistently as intended. Results are documented." }, { "requirementID": "MEASURE 4.3", "requirementText": "Measurable performance improvements or declines based on consultations with relevant AI actors, including affected communities, and field data about context-relevant risks and trustworthiness characteristics are identified and documented." } ] }, "IR 8596: Cybersecurity Framework Profile for Artificial Intelligence (Cyber AI Profile): NIST Community Profile": { "link": "https://csrc.nist.gov/pubs/ir/8596/iprd", "requirements": [ { "requirementID": "PR.PS-01", "requirementText": "Configuration management practices are established and applied" }, { "requirementID": "RS.AN-06", "requirementText": "Actions performed during an investigation are recorded, and the records’ integrity and provenance are preserved" } ] }, "SP 800-218A": { "link": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirements": [ { "requirementID": "PO.3.3", "requirementText": "Configure tools to generate artifacts of their support of secure software development practices as defined by the organization." }, { "requirementID": "PS.2.1", "requirementText": "Make software integrity verification information available to software acquirers.\n\nGenerate and provide cryptographic hashes or digital signatures for an AI model and its components, artifacts, and documentation.\n\nProvide digital signatures for AI model changes." }, { "requirementID": "PS.3.1", "requirementText": "Securely archive the necessary files and supporting data (e.g., integrity verification information, provenance data) to be retained for each software release.\n\nPerform versioning and tracking for infrastructure tools (e.g., pre-processing, transforms, collection) that support dataset creation and model training.\n\nInclude documentation of the justification for AI model selection in the retained information.\n\nInclude documentation of the entire training process, such as data preprocessing and model architecture." }, { "requirementID": "PW.1.2", "requirementText": "Track and maintain the software’s security requirements, risks, and design decisions." }, { "requirementID": "PW.3.2", "requirementText": "Track the provenance, when known, of all training, testing, fine-tuning, and aligning data used for an AI model, and document which data do not have known provenance." }, { "requirementID": "PW.9.2", "requirementText": "Implement the default settings (or groups of default settings, if applicable), and document each setting for software administrators." } ] } }, "OWASP": { "LLM Top 10": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirements": [ { "requirementID": "LLM01: Prompt Injection - 2", "requirementText": "Specify clear output formats, request detailed reasoning and source citations, and use deterministic code to validate adherence to these formats" }, { "requirementID": "LLM08: Vector and Embedding Weaknesses - 3", "requirementText": "When combining data from different sources, thoroughly review the combined dataset. Tag and classify data within the knowledge base to control access levels and prevent data mismatch errors." } ] }, "OWASP Model Context Protocol (MCP) Top 10": { "link": "https://owasp.org/www-project-mcp-top-10/", "requirements": [ { "requirementID": "MCP02:2025 - Privilege Escalation via Scope Creep - 7", "requirementText": "Strong Change Management & Audit Trails All permission changes must be tracked, reviewed, and linked to a change request or ticket. Keep immutable, tamper-evident logs tying actions to agent identity and session." }, { "requirementID": "MCP10:2025 – Context Injection & Over-Sharing - 7", "requirementText": "Context Access Logging\nLog:\nAgent ID\nContext ID\nRead/write events\nTTL + purge events Integrate context logs into SIEM/XDR for monitoring." } ] } }, "Personal Data Protection Commission Singapore (PDPC)": { "Model Artificial Intelligence Governance Framework Second Edition": { "link": "https://www.pdpc.gov.sg/-/media/files/pdpc/pdf-files/resource-for-organisation/ai/sgmodelaigovframework2.pdf", "requirements": [ { "requirementID": "Data for Model Development - a) Understanding the lineage of data", "requirementText": "This means knowing where the data originally came from, how it was collected, curated and moved within the organisation, and how its accuracy is maintained over time. Data lineage can be represented visually to trace how the data moves from its source to its destination, how the data gets transformed along the way, where it interacts with other data, and how the representations change. There are three types of data lineage:\ni. Backward data lineage looks at the data from its end-use and backdating it to its source.\nii. Forward data lineage begins at the data’s source and follows it through to its end-use.\niii. End-to-end data lineage combines the two and looks at the entire solution from both the data’s source to its end-use and from its end-use to its source. Keeping a data provenance record allows an organisation to ascertain the quality of the data based on its origin and subsequent transformation, trace potential sources of errors, update data, and attribute data to their sources.\nIn some instances, the origin of data could be difficult to establish. One example could be datasets obtained from a trusted third-party which may have commingled data from multiple sources. It would be prudent for organisations to assess the risks of using such data and manage them accordingly." }, { "requirementID": "ALGORITHM AND MODEL - a)", "requirementText": "Model training and selection are necessary for developing an intelligent system (i.e. a system that contains AI technologies). Documenting how the model training and selection processes are conducted, the reasons for which decisions are made, and measures taken to address identified risks will enable the organisation to provide an account of the decisions subsequently. \n\nIn this regard, the field of Automated Machine Learning aims to automate a significant portion of machine learning workflows, including feature engineering, feature selection, model selection and hyper-parameter tuning. Organisations using these types of tools can consider the transparency, explainability and traceability of the automated machine learning approach, as well as the models selected." }, { "requirementID": "ALGORITHM AND MODEL - b)", "requirementText": "Incorporating descriptions of the solutions’ design and expected behaviour into product or service descriptions and system technical specifications documentation demonstrates accountability to individuals and/or regulators. This could also include design decisions in relation to why certain features, attributes or models are selected in place of others. These steps can help provide greater clarity on an AI model by giving understandable and digestible insights into how the model operates.\n\nWhere an organisation’s AI system was obtained or procured from a third-party AI solution provider, the organisation can consider requesting assistance from the AI solution provider as they may be better placed to explain how the solution functions." } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirements": [ { "requirementID": "8.2.2", "requirementText": "Maintain, monitor, document, and review of the Al Models that have been deployed." }, { "requirementID": "10.1", "requirementText": "An Entity must develop and maintain an updated Register of information on all its Al Systems arrangements." }, { "requirementID": "10.7", "requirementText": "The Register of the Entity must include for each system:\n- The classification of high-risk or not high-risk.\n- The role of an Entity, that is, whether it a User of the system or a Provider of the system.\n- A category assigned by an Entity that reflects the nature or functional use of the Al System.\n- The Human Oversight protocol used.\nIn addition, if the Al System is purchased or licensed or outsourced:\n- The name of the Provider and any outsourcer.\n- Third party supplier assessment.\n- The country of registration, local corporate registration number and LEI (where applicable).\n- Registered address and relevant contact details.\n- Name of the parent company (where applicable).\n- Governing law of the Al licensing or purchase arrangement.\n- The creation date of the system and all subsequent upgrades.\n- The contract start date, as applicable.\n- The next contract renewal date.\n- For any high-risk system:\n - Assess the Al's substitutability as easy, difficult, or impossible.\n - Identify an alternate service Provider, (where applicable).\n- An Entity must maintain a detailed description of all High-Risk Al Systems including Life Cycle (development history, data used, testing, tracking).\n- The date of the most recent risk assessment or audit of all High-Risk Al Systems together with a summary of the main results, and the date of the next planned risk assessment/ audit." }, { "requirementID": "10.8", "requirementText": "An Entity must maintain a detailed description of all High-Risk Al Systems including Life Cycle (development history, data used, testing, tracking)." }, { "requirementID": "10.9", "requirementText": "The date of the most recent risk assessment or audit of all High-Risk Al Systems together with a summary of the main results, and the date of the next planned risk assessment/ audit." }, { "requirementID": "12.5", "requirementText": "An Entity must obtain approval from its Board of Directors to outsource any function in relation to the use of Al and must document it." }, { "requirementID": "16.1", "requirementText": "Where an Entity is the User or Provider of a High-Risk Al System it must put a framework in place that ensures compliance with this guideline. It must be documented in a systematic and orderly manner in the form of written policies, procedures, and instructions, and must include at least the following aspects:\n- A framework for regulatory compliance, including compliance with QCB procedures for how to manage or modify High-Risk Al Systems.\n- Techniques, procedures, and systematic actions to be used for the design, design control and design verification of a High-Risk Al System.\n- Techniques, procedures, and systematic actions to be used for the development, quality control and quality assurance of the High-Risk Al System.\n- Examination, testing and validation procedures to be carried out before, during and after the development of the High-Risk Al System, and the frequency of post-launch reviews.\n- Systems and procedures for data management, including data collection, data analysis, data labelling, data storage, data filtration, data mining, data aggregation, data retention and any other operation regarding the data that is performed before and for the purposes of the placing on the market or putting into service of High-Risk Al Systems.\n- The setting-up, implementation and maintenance of a Monitoring System.\n- Systems and procedures for record keeping of all relevant documentation and information.\n- Resource management, including security of supply-related measures.\n- An accountability framework setting out the responsibilities of the management and other staff." }, { "requirementID": "18.1", "requirementText": "An Entity should maintain documentation outlining the design of the Al Model, whether in the role of Provider or User, including but not limited to, where applicable:\n- The Input Data source and data description (types and use of data) as per the Al data governance above.\n- The data quality checks and data transformations conducted.\n- Reasons and justifications for specific model design and development choices.\n- Methodology or numerical analyses and calculations conducted.\n- Results and expected outcomes.\n- Quantitative evaluation and testing metrics used to determine soundness of the model and its results.\n- Model usage and implementation.\n- Form and frequency of model validation, monitoring and review.\n- Assumptions or limitations of the model with justifications." }, { "requirementID": "19.3.2", "requirementText": "Develop and maintaining design documentation." } ] } }, "SANS": { "Critical AI Security Guidelines": { "link": "https://sansorg.egnyte.com/dl/bvkYQxrW8QMj", "requirements": [ { "requirementID": "3.1 Assess Model Hosting Options: Local vs. SaaS Models", "requirementText": "When weighing where and how to host AI solutions, be sure to think carefully about and codify legal requirements in any contracts. For example, will your data ever be used or retained by the provider for training or refining a model? If the provider claims that they will not store or use your data, what steps have been taken to prevent your data from being logged when sent to and processed by the API endpoint? How are these logs controlled? How long are they stored? Who has access to them?" } ] } }, "SDAIA (Saudi Arabia)": { "AI Ethics Principles": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Documents/ai-principles.pdf", "requirements": [ { "requirementID": "Principle 5 – Reliability & Safety - Plan and Design - 4", "requirementText": "The documentation standards are essential to track the evolution of the system, foresee possible risks and fix vulnerabilities." }, { "requirementID": "Principle 6 – Transparency & Explainability - Prepare Input Data - 1", "requirementText": "The data sets and the processes that yield the AI system’s decision should be documented to the best possible standard to allow for traceability and an increase in transparency." }, { "requirementID": "Principle 7 – Accountability & Responsibility - Prepare Input Data - 3", "requirementText": "The documentation of the process is necessary for auditing and risk mitigation. Data must be properly acquired, classified, processed, and accessible to ease human intervention and control at later stages when needed." } ] } }, "Smart Dubai (UAE)": { "AI Ethics Principles & Guidelines": { "link": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirements": [ { "requirementID": "1.3.1.1", "requirementText": "For AI systems which inform significant decisions, especially those with the potential to cause loss, harm or damage, AI developer organisations should consider building in traceability, i.e. the ability to trace the key factors leading to any specific decision." }, { "requirementID": "1.3.1.2", "requirementText": "To facilitate the above, AI developer organisations and AI operator organisations should consider documenting the following information during the design, development and deployment phases, and retaining this documentation for a length of time appropriate to the decision type or industry:\n• the provenance of the training data, the methods of collection and treatment, how the data was moved, and measures taken to maintain its accuracy over time;\n• the model design and algorithms employed, and;\n• changes to the codebase, and authorship of those changes" } ] } }, "World Economic Forum": { "Presidio AI Framework: Towards Safe Generative AI Models": { "link": "https://www3.weforum.org/docs/WEF_Presidio_AI%20Framework_2024.pdf", "requirements": [ { "requirementID": "Transparent documentation and use restriction", "requirementText": "Transparent documentation is a collection of details (decisions, choices and processes) about the AI model, including the data. It mitigates the risk of lack of transparency, and therefore empowers downstream adapters and users to understand the model’s limitations, evaluate its impact and make decisions on model use. This guardrail increases the auditability of the model and helps advance policy initiatives." } ] } } }, "Principle 9": { "CEN/CENELEC": { "prEN 40000-1-2: Cybersecurity requirements for products with digital elements - Part 1-2: Principles for cyber resilience": { "link": "https://genorma.com/en/standards/pren-40000-1-2", "requirements": [ { "requirementID": "7.5", "requirementText": "Cybersecurity Verification and validation" } ] } }, "Central Bank of the UAE": { "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E": { "link": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirements": [ { "requirementID": "3. Fairness/Non-Discrimination and Ethics - c", "requirementText": "AI deployed should be subject to periodic testing, i.e. once a year or each time a model is upgraded, materially changed or a new one is introduced, to identify and remediate undue/unintended embedded biases or discriminatory outcomes. The deployment of AI should reflect the institution’s ethical standards and code of conduct. Decisions made or supported by AI should be consistent with the duty to act honestly, fairly and in the best interests of consumers. " }, { "requirementID": "5. Data Quality, Privacy and Security - d", "requirementText": "Robustness and safety should be integral to AI development. AI should be subject to stress testing and validation to ensure they operate reliably under a range of scenarios and do not produce unsafe or unintended outputs. Institutions should incorporate operational resilience measures—such as redundancy, contingency planning and incident response—to minimise disruption or harm to consumers from system failures or cyber-attacks. " }, { "requirementID": "8. Integration with Existing Frameworks - c", "requirementText": "Where an LFI is developing AI internally, it should consider the use of third-party independent reviews to check suitability, security and reliability. " } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirements": [ { "requirementID": "3.3 Conduct Thorough AI Testing and Evaluation", "requirementText": "Operators should initially conduct tests of the AI system on infrastructure specifically designed for testing." } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "A&A-02", "requirementText": "Conduct independent audit and assurance assessments according to relevant standards at least annually." }, { "requirementID": "A&A-03", "requirementText": "Perform independent audit and assurance assessments in response to significant changes or emerging risks and according to risk-based plans and policies." }, { "requirementID": "A&A-05", "requirementText": "Define and implement an Audit Management process aligned with global auditing standards, to support audit planning, risk analysis, security control assessment, conclusion, remediation schedules, report generation, and review of past reports and supporting evidence." }, { "requirementID": "AIS-05", "requirementText": "Implement a testing strategy, including criteria for acceptance of new information systems, upgrades and new versions, which provides application security assurance and maintains compliance while meeting organizational delivery goals. Automate when applicable and possible." }, { "requirementID": "CCC-02", "requirementText": "Establish, maintain and implement a defined quality change control, approval and testing process incorporating baselines, testing, and release standards." }, { "requirementID": "CEK-09", "requirementText": "Audit encryption and key management systems, policies, and processes with a frequency that is proportional to the risk exposure of the system with audit occurring preferably continuously but at least annually and after any security event(s)." }, { "requirementID": "DCS-14", "requirementText": "Secure, monitor, maintain, and test utilities services for continual effectiveness at planned intervals." }, { "requirementID": "GRC-11", "requirementText": "Regularly evaluate AI systems, models, datasets & algorithms for bias and fairness to ensure compliance with ethical standards." }, { "requirementID": "SEF-04", "requirementText": "Follow a structured approach to evaluate the effectiveness of incident response plans at planned intervals or upon significant changes." }, { "requirementID": "TVM-06", "requirementText": "Define, implement and evaluate processes, procedures and technical measures for the periodic performance of penetration testing by independent third parties." } ] } }, "CoSAI": { "AI Incident Response Framework": { "link": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirements": [ { "requirementID": "3.3.3. Containment, Eradication, and Recovery Phase - Eradication Procedures - Verification Testing", "requirementText": "• Penetration testing\n• Attack reproduction attempts\n• Red-team exercises\n• Issue verification" } ] }, "Establish Risks and Controls for the AI Supply Chain": { "link": "https://github.com/cosai-oasis/ws1-supply-chain/blob/main/risks-and-controls-for-the-ai-supply-chain-v1.md", "requirements": [ { "requirementID": "3.1.2 Model Training - Compromised Evaluation", "requirementText": "Non-reproducible evaluation processes allowing fraudulent results to promote malicious models:\nTamper-proof documentation of evaluation protocols and results" }, { "requirementID": "3.1.4 AI Infrastructure Supply Chain - Serving Infrastructure Compromise", "requirementText": "Vulnerabilities in model serving platforms that may affect multiple hosted models:\nEvaluate isolation mechanisms, request penetration testing reports, implement client-side verification" }, { "requirementID": "3.1.4 AI Infrastructure Supply Chain - Development Environment Artifacts", "requirementText": "Backdoors or vulnerabilities introduced during model development persisting into production:\nRequest development security documentation, perform independent model validation and testing" }, { "requirementID": "3.2.1 Supply Chain Security for Data - Data Leakage (output)", "requirementText": "Unintended exposure of sensitive information via model outputs:\n• Implement robust output filtering mechanisms\n• Regularly audit and monitor model outputs\n• Establish rate limiting on similar queries" } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "2.4.4", "requirementText": "The entity shall ensure the security of AI/ML systems through comprehensive testing and validation, providing assurance of their resilience against potential threats." }, { "requirementID": "3.1.4 Change Management and Reporting - 3", "requirementText": "All manual changes should be tested in a secure environment before deployment to identify and address any unexpected impacts or vulnerabilities introduced by the change." }, { "requirementID": "3.2.3 Patch Management - 3", "requirementText": "All patches and updates should undergo testing and verification in a controlled environment before deployment in production systems to prevent unintended impacts on system performance and availability." }, { "requirementID": "3.2.4 Vulnerability Management for AI/ML Systems - 2", "requirementText": "The entity should employ an appropriate combination of testing tools, methodologies, and external audits to identify vulnerabilities in AI/ML infrastructure and applications." }, { "requirementID": "3.2.5 Application Security - 2", "requirementText": "The entity should perform security testing, including code reviews and static/dynamic analysis, to identify and remediate potential security flaws in AI/ML applications prior to deployment." }, { "requirementID": "3.2.6 Network Security for AI/ML Infrastructure - 6", "requirementText": "The entity should routinely conduct vulnerability assessments and penetration tests on the AI/ML network infrastructure to identify and remediate potential security weaknesses." }, { "requirementID": "3.4.4 Testing and Validation of AI/ML Systems - 1", "requirementText": "The entity should establish and maintain a robust framework for the testing and validation of AI/ML systems, ensuring their accuracy, reliability, and performance meet defined criteria." }, { "requirementID": "3.4.4 Testing and Validation of AI/ML Systems - 2", "requirementText": "All AI/ML systems should undergo rigorous testing before deployment to verify their functionality, security, and resilience against potential threats." }, { "requirementID": "3.4.4 Testing and Validation of AI/ML Systems - 3", "requirementText": "The testing process should consider various scenarios and inputs, including edge cases, to evaluate the AI/ML systems' behavior under different conditions." }, { "requirementID": "3.5.2 Defending Against AI/ML Attacks - 3", "requirementText": "Regular security testing, including penetration testing and red teaming exercises, should be conducted on AI/ML systems to identify potential weaknesses and implement necessary defenses." } ] } }, "Databricks": { "The Databricks AI Security Framework": { "link": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirements": [ { "requirementID": "DASF 38: Platform security — penetration testing, red teaming, bug bounty and vulnerability management\n", "requirementText": "Mitigating attacks on infrastructure hosting AI services, including AI red teaming for large language models, is crucial for safe model development and deployment. Regular security and penetration testing help identify and address infrastructure vulnerabilities before attackers can exploit them.\nDatabricks operates a formal, documented vulnerability management program overseen by the Chief Security Officer (CSO). The program is management-approved, reviewed annually, and communicated to all relevant internal parties. The policy mandates that vulnerabilities be addressed based on severity: critical vulnerabilities within 14 days, high-severity vulnerabilities within 30 days, and medium-severity vulnerabilities within 60 days.\nAI red teaming, especially for large language models, is an essential component of ensuring model safety and security. Databricks conducts regular AI red teaming on models and systems developed internally." }, { "requirementID": "DASF 45: Evaluate models", "requirementText": "Model evaluation is a critical component of the machine learning lifecycle. It provides data scientists with the tools to measure, interpret and explain the performance of their models. MLflow plays a critical role in accelerating model development by offering insights into the reasons behind a model’s performance and guiding improvements and iterations. MLflow offers many industry-standard native evaluation metrics for classical machine learning algorithms and LLMs, and also facilitates the use of custom evaluation metrics." }, { "requirementID": "DASF 47: Compare LLM outputs on set prompts\n", "requirementText": "New, no-code visual tools allow users to compare models’ output based on set prompts, which are automatically tracked within MLflow. With integration into Mosaic AI Model Serving, customers can deploy the best model to production. The AI Playground is a chat-like environment where you can test, prompt and compare LLMs." }, { "requirementID": "DASF 49: Automate LLM evaluation\n", "requirementText": "The “LLM-as-a-judge” feature in MLflow 2.8 automates LLM evaluation, offering a practical alternative to human judgment. It’s designed to be efficient and cost-effective, maintaining consistency with human scores. This tool supports various metrics, including standard and customizable GenAI metrics, and allows users to select an LLM as a judge and define specific grading criteria." }, { "requirementID": "DASF 66: Use Human-in-the-loop feedback", "requirementText": "End-user feedback is invaluable for understanding how your GenAI application performs in real-world scenarios. MLflow provides tools to capture, store, and analyze feedback directly from the users of your deployed applications." } ] } }, "ENISA": { "Multilayer Framework for Good Cybersecurity Practices for AI": { "link": "https://www.enisa.europa.eu/sites/default/files/publications/Multilayer%20Framework%20for%20Good%20Cybersecurity%20Practices%20for%20AI.pdf", "requirements": [ { "requirementID": "Security testing", "requirementText": "Security testing of AI has some commonalities with security testing of traditional systems, but also provides new challenges and requires different approaches, due to:\n• significant differences between subsymbolic AI and traditional systems that have strong implications on their security and on how to test their security properties;\n• non-determinism that may result from self-learning, i.e. AI-based systems may evolve over time and as a consequence, security properties may degrade;\n• the test oracle problem, where assigning a test verdict is different and more difficult for AI-based systems, since not all expected results are known a priori;\n• data-driven algorithms, where in contrast to traditional systems, (training) data forms the behaviour of subsymbolic AI." }, { "requirementID": "From the lab to the market 3", "requirementText": "Do you have/promote testing environments, like sandboxes/cyber ranges/simulation platforms to test and evaluate AI vulnerabilities before market? How?" }, { "requirementID": "Infrastructure 2", "requirementText": "Have you specified/defined measurements and KPIs which the AI stakeholders can use to assess the appropriateness of the controls undertaken?" }, { "requirementID": "Regulation 2", "requirementText": "Have national auditors and certification and accreditation bodies been established for assessing the security of the AI systems?" }, { "requirementID": "Regulation 3", "requirementText": "How do you evaluate security of the AI systems (e.g. via conformity assessment, certification, standards compliance, risk assessment, etc.)?" }, { "requirementID": "Regulation 4", "requirementText": "What are the obligations you have imposed for testing, risk management, documentation and human oversight throughout the AI systems’ life cycle to ensure continuous data and training model integrity?" } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.2.5-1", "requirementText": "Developers shall ensure that all models, applications and systems that are released to System Operators and/or End-users have been tested as part of a security assessment process." }, { "requirementID": "Provision 5.2.5-2", "requirementText": "System Operators shall conduct testing prior to the system being deployed with support from Developers." }, { "requirementID": "Provision 5.2.5-2.1", "requirementText": "For security testing, System Operators and Developers should use independent security testers with technical skills relevant to their AI systems." }, { "requirementID": "Provision 5.2.5-3", "requirementText": "Developers should ensure that the findings from the testing and evaluation are shared with System Operators, to inform their own testing and evaluation." }, { "requirementID": "Provision 5.2.5-4", "requirementText": "Developers should evaluate model outputs to ensure they do not allow System Operators or End-users to reverse engineer non-public aspects of the model or the training data." }, { "requirementID": "Provision 5.2.5-4.1", "requirementText": "Additionally, Developers should evaluate model outputs to ensure they do not provide System Operators or End-users with unintended influence over the system." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.2.5-1", "requirementText": "\"Developers shall ensure that all models, applications, and systems that are released have been tested as part of a security assessment process.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout rigorous security assessments by qualified experts understanding AI and classic risks, AI systems can be vulnerable to exploits, unauthorized access, or data breaches, leading to potential data loss, manipulation, or misuse of the AI model.\n\nExample Measures/Controls 1:\nImplement Security Assessment Processes for All Releases: Establish a mandatory security assessment process for all models, applications, and systems prior to release, covering areas like access control, data integrity, and adversarial AI attacks. Scope testing based on the threats identified in threat models.\n\nExample Measures/Controls 2:\nUse Offensive Security Assessments with Penetration Testing and Red Teaming: Use periodic penetration testing to evaluate the AI system's resilience and red teaming for models." }, { "requirementID": "Provision 5.2.5-2", "requirementText": "\"System Operators shall conduct testing prior to the system being deployed with support from Developers.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout pre-deployment testing, systems can fail to meet operational and security requirements, leading to compromised performance, reduced reliability, or security vulnerabilities once deployed.\n\nExample Measures/Controls:\nImplement Comprehensive Pre-Deployment Testing: Compliment development-time testing with running benchmarking emulation suites covering functional, performance, and security tests to confirm that the system meets intended requirements before deployment. These can either be in-house or independent third-party benchmarks. Integrate these tests into the development pipeline to ensure issues are identified and resolved before release." }, { "requirementID": "Provision 5.2.5-2.1", "requirementText": "\"For security testing, System Operators and Developers should use independent security testers with technical skills relevant to their AI systems.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout independent security testing, systems can overlook critical vulnerabilities, especially those requiring specialized expertise, increasing the risk of undetected flaws and potential exploitation.\n\nExample Measures/Controls:\nEngage Independent Security Testers for Pre-Deployment Testing: Use independent security testers with AI expertise to validate security measures, providing an unbiased review of system security." }, { "requirementID": "Provision 5.2.5-3", "requirementText": "\"Developers should ensure that the findings from the testing and evaluation are shared with System Operators, to inform their own testing and evaluation.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout access to previous testing findings, System Operators can lack critical insights into known vulnerabilities or limitations, and mitigations that cannot be seen as adequate by operators leading to potential gaps in security coverage.\n\nExample Measures/Controls:\nEstablish a Process for Sharing Testing Results: Create a standardized process for sharing all relevant testing results and evaluation findings with System Operators of testing reports and ensure timely access for all relevant stakeholders." }, { "requirementID": "Provision 5.2.5-4", "requirementText": "\"Developers should evaluate model outputs to ensure they do not allow System Operators or End-users to reverse engineer non-public aspects of the model or the training data.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout adequate output evaluation, Operators or End-users can reverse engineer model internals or correlate outputs with sequences of predesigned inputs, enabling them to reconstruct the model or training data. This can erode commercial advantage, allow the creation of shadow models, or facilitate attacks, even affecting \"open source\" models if only parts, such as architecture or weights, are released.\n\nExample Measures/Controls 1:\nConduct Security Reviews for Output Sensitivity: Engage with AI experts to evaluate model outputs and identify any information that could reveal internal model structures, ensuring sensitive aspects remain non-public.\n\nExample Measures/Controls 2:\nIntegrate Adversarial Testing and Robustness Evaluation: Implement adversarial testing to assess the resilience of AI systems against attempts to reverse engineer or manipulate model outputs. This involves simulating attacks that exploit output data to uncover model weaknesses or extract sensitive information. Open-source tools like ART and TextAttack can help to develop and run these tests." }, { "requirementID": "Provision 5.2.5-4.1", "requirementText": "\"Additionally, Developers should evaluate model outputs to ensure they do not provide System Operators or End-users with unintended influence over the system.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout evaluating for unintended influence, Operators or End-users can exploit system behaviour to align outputs with personal or malicious goals, leading to bias, misuse, or compromised trust in the system.\n\nExample Measures/Controls:\nImplement Safeguards Against Manipulation: Set controls to prevent Operators or End-users from adjusting inputs in ways that could intentionally influence the AI system's outcomes." } ] }, "TR 104 222 - Securing Artificial Intelligence; Mitigation Strategy Report": { "link": "https://www.etsi.org/deliver/etsi_tr/104200_104299/104222/01.02.01_60/tr_104222v010201p.pdf", "requirements": [ { "requirementID": "6.2.2 - 8", "requirementText": "Adversarial testing is closely related to adversarial training described earlier. Adversarial testing consists of testing the robustness of a trained model with respect to adversarial examples. These attacks can be created using a variety of methods, in particular projected gradient descent. Adversarial testing allows quantifying the robustness of a model to the considered attacks. If the testing results show that the robustness of the model is low, the model can be hardened more thoroughly, using adversarial training." } ] } }, "EU ": { "EU AI Act": { "link": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirements": [ { "requirementID": "9.7 Risk Management System", "requirementText": "Testing procedures may include testing in real-world conditions in accordance with Article 60." }, { "requirementID": "9.8 Risk Management System", "requirementText": "The testing of high-risk AI systems shall be performed, as appropriate, at any time throughout the development process, and, in any event, prior to their being placed on the market or put into service. Testing shall be carried out against prior defined metrics and probabilistic thresholds that are appropriate to the intended purpose of the high-risk AI system." }, { "requirementID": "15.2 Accuracy, Robustness and Cybersecurity", "requirementText": "To address the technical aspects of how to measure the appropriate levels of accuracy and robustness set out in paragraph 1 and any other relevant performance metrics, the Commission shall, in cooperation with relevant stakeholders and organisations such as metrology and benchmarking authorities, encourage, as appropriate, the development of benchmarks and measurement methodologies." } ] } }, "European Commission": { "Ethics guidelines for trustworthy AI": { "link": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirements": [ { "requirementID": "1.7.1 Auditability", "requirementText": "Auditability entails the enablement of the assessment of algorithms, data and design processes. This does not necessarily imply that information about business models and intellectual property related to the AI system must always be openly available. Evaluation by internal and external auditors, and the availability of such evaluation reports, can contribute to the trustworthiness of the technology. In applications affecting fundamental rights, including safety-critical applications, AI systems should be able to be independently audited." }, { "requirementID": "2.1.4 Testing and Validating", "requirementText": "Testing and validation of the system should occur as early as possible, ensuring that the system behaves as intended throughout its entire life cycle and especially after deployment. It should include all components of an AI system, including data, pre-trained models, environments and the behaviour of the system as a whole. The testing processes should be designed and performed by an as diverse group of people as possible. Multiple metrics should be developed to cover the categories that are being tested for different perspectives. Adversarial testing by trusted and diverse “red teams” deliberately attempting to “break” the system to find vulnerabilities, and “bug bounties” that incentivise outsiders to detect and responsibly report system errors and weaknesses, can be considered. Finally, it must be ensured that the outputs or actions are consistent with the results of the preceding processes, comparing them to the previously defined policies to ensure that they are not violated." }, { "requirementID": "2.1.5 Quality of Service Indicators", "requirementText": "Appropriate quality of service indicators can be defined for AI systems to ensure that there is a baseline understanding as to whether they have been tested and developed with security and safety considerations in mind. These indicators could include measures to evaluate the testing and training of algorithms as well as traditional software metrics of functionality, performance, usability, reliability, security and maintainability." } ] } }, "Federal Office for Information Security": { "AI Security Concerns in a Nutshell": { "link": "https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/KI/Practical_Al-Security_Guide_2023.pdf?__blob=publicationFile&v=5", "requirements": [ { "requirementID": "5.3 Defending against Poisoning and Backdoor Attacks - Search for Triggers", "requirementText": "The triggers used for backdoors rely on logical shortcuts between the target class and the input. To find a shortcut, one must determine the minimal input change required to shift the model’s prediction. If such a change is minimal, a backdoor may have been found [20]. Another method for the image domain is to randomly mask parts of an input image and examine how the model‘s prediction changes. If the input image contains a trigger, masking it will change the model’s prediction [21]." } ] } }, "Google": { "Secure AI Framework": { "link": "https://www.saif.google/secure-ai-framework", "requirements": [ { "requirementID": "Model and Data Access Controls", "requirementText": "Minimize internal access to models, weights, datasets, etc. in storage and in production use." }, { "requirementID": "Red Teaming", "requirementText": "Identify security and privacy improvements through self-driven adversarial attacks on AI infrastructure and products." }, { "requirementID": "Product Governance", "requirementText": "Validate that all AI models and products meet the established security and privacy requirements." } ] } }, "ICO": { "Guidance on the AI Auditing Framework - Draft guidance for consultation ": { "link": "https://ico.org.uk/media2/about-the-ico/consultations/2617219/guidance-on-the-ai-auditing-framework-draft-for-consultation.pdf", "requirements": [ { "requirementID": "Detective Controls - 2", "requirementText": "Regularly test, assess and evaluate the effectiveness of any security measures they have put in place (eg through techniques such as penetration testing)." } ] } }, "IMDA": { "Model AI Governance Framework for Agentic AI": { "link": "https://www.imda.gov.sg/-/media/imda/files/about/emerging-tech-and-research/artificial-intelligence/mgf-for-agentic-ai.pdf", "requirements": [ { "requirementID": "2.3.2 Before deploying, test agents", "requirementText": "Organisations should test agents for safety and security before deployment. This provides confidence that the agents work as expected and controls are effective. Best practices on software and LLM testing are still relevant, such as unit and integration testing for software systems, as well as selecting representative datasets, and useful metrics and evaluators for LLM testing. Organisations can refer to previous guidance, such as the Starter Kit for testing of LLM-based apps for safety and reliability." } ] } }, "ISO": { "42001:2023 - Information technology — Artificial intelligence — Management system": { "link": "https://www.iso.org/standard/42001", "requirements": [ { "requirementID": "9.2", "requirementText": "Internal audit" }, { "requirementID": "9.3.3", "requirementText": "Management review results" } ] } }, "ISO/IEC": { "TS 42119-2:2025": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:ts:42119:-2:ed-1:v1:en", "requirements": [ { "requirementID": "5.4", "requirementText": "Risk-based testing" }, { "requirementID": "5.5", "requirementText": "Test processes" }, { "requirementID": "5.6", "requirementText": "Test documentation" }, { "requirementID": "5.7", "requirementText": "Testing stakeholders" }, { "requirementID": "7.1", "requirementText": "Introduction to test approaches" }, { "requirementID": "7.2", "requirementText": "Test levels" }, { "requirementID": "7.3", "requirementText": "Test types" }, { "requirementID": "7.4", "requirementText": "Test design techniques and measures" }, { "requirementID": "A.2", "requirementText": "Purpose of testing" }, { "requirementID": "A.3", "requirementText": "Software testing in context" }, { "requirementID": "A.4", "requirementText": "Static and dynamic testing" }, { "requirementID": "A.5", "requirementText": "Test processes (summarized from ISO/IEC/IEEE 29119-2)" } ] } }, "METI (Japan)": { "Governance Guidelines for Implementation of AI Principles": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20220128_2.pdf", "requirements": [ { "requirementID": "Action Target 6-1", "requirementText": "Companies that develop and operate AI systems should, under the leadership of top management, conduct re-evaluations, update their understanding, obtain new points of views, or take other relevant actions with respect to Action Targets 1-1 through 1.3, in a timely manner. When implementing Action Target 5-2, they should also consider obtaining opinions not only for the current AI management system and the operation of such system, but also conducive to review of entire AI governance, including analyses of conditions and risks." } ] } }, "MIC/METI (Japan)": { "AI Guidelines for Business": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirements": [ { "requirementID": "Safety - 1 (a)", "requirementText": "Ensure that the AI system/service is sufficiently fulfilling the requirements, including the accuracy of outputs (reliability)." }, { "requirementID": "Safety - 1 (b)", "requirementText": "Ensure that the performance level is maintained under various circumstances, and that grossly incorrect judgments are not made for irrelevant events (robustness)." } ] } }, "Microsoft": { "Cloud Adoption Framework - Secure AI": { "link": "https://learn.microsoft.com/en-us/azure/cloud-adoption-framework/scenarios/ai/secure", "requirements": [ { "requirementID": "Discover AI security risks\n3 - Test AI models for security vulnerabilities", "requirementText": "AI models contain unique vulnerabilities like data leakage, prompt injection, and model inversion that attackers can exploit. Real-world testing uncovers risks that static reviews can't detect. Test models for vulnerabilities using data-loss-prevention techniques and adversarial simulations, and red team both generative AI and nongenerative AI models to simulate real attacks." } ] }, "Responsible AI Standard": { "link": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirements": [ { "requirementID": "A3.4", "requirementText": "Document an evaluation plan for each of the performance metrics and error types." }, { "requirementID": "A3.5", "requirementText": "Use the methods defined in requirement A3.4 to conduct evaluations. Document the pre-release results of the evaluations. Determine and document how often ongoing evaluation should be conducted to continue supporting this Goal." }, { "requirementID": "A5.6", "requirementText": "Conduct evaluations defined by requirement A5.4 using a near-release version of the system. Document the results." }, { "requirementID": "T1.5", "requirementText": "Conduct evaluations defined by requirement T1.3. Document the pre-release results of the evaluations. Determine and document how often ongoing evaluation should be conducted to continue supporting this Goal." }, { "requirementID": "T3.4", "requirementText": "Define and document Responsible Release Criteria to achieve this Goal." }, { "requirementID": "T3.5", "requirementText": "Conduct evaluations defined by requirement T3.3. Document the pre-release results of the evaluations. Determine and document how often ongoing evaluation should be conducted to continue supporting this goal." }, { "requirementID": "F1.2", "requirementText": "Evaluate all data sets to assess inclusiveness of identified demographic groups and collect data to close gaps. Document this process and its results." }, { "requirementID": "F1.3", "requirementText": "Define and document the evaluation that you will perform to support this Goal. Include:\n1) any system components to be evaluated, in addition to the whole system,\n2) the metrics to be used to evaluate the system components and the whole system, and\n3) a description of the data set to be used for this evaluation." }, { "requirementID": "F1.4", "requirementText": "Define and document Responsible Release Criteria to achieve this Goal, as follows:\nFor each metric, document:\n1) any target minimum performance level for all groups, and\n2) the target maximum (absolute or relative) performance difference between groups." }, { "requirementID": "F1.5", "requirementText": "Evaluate the system according to the defined Responsible Release Criteria." }, { "requirementID": "F2.2", "requirementText": "Evaluate all data sets to assess inclusiveness of identified demographic groups and collect data to close any gaps. Document this process and its results." }, { "requirementID": "F2.3", "requirementText": "Define and document the evaluation that you will perform to support this Goal. Include:\n1) any system components to be evaluated, in addition to the whole system,\n2) the metrics to be used to evaluate the system components and the whole system, and\n3) the data set to be used for this evaluation." }, { "requirementID": "F2.4", "requirementText": "Define and document Responsible Release Criteria to achieve this Goal, as follows:\nFor each metric, document the target maximum difference (absolute or relative) between the rates at which resources and opportunities are allocated to groups." }, { "requirementID": "F2.5", "requirementText": "Evaluate the system according to the defined Responsible Release Criteria." }, { "requirementID": "F3.2", "requirementText": "Define and document any system components to be evaluated, in addition to the whole system." }, { "requirementID": "F3.3", "requirementText": "Define and document a plan to evaluate the system components and the whole system for risks of stereotyping, demeaning, and erasing the prioritized identified demographic groups." }, { "requirementID": "F3.4", "requirementText": "Evaluate the system according to the plan defined in requirement F3.3." }, { "requirementID": "RS1.2", "requirementText": "Evaluate training and test data sets to ensure that they include representation of the intended uses, operational factors, and an appropriate range of settings for each factor. Document the evaluation." }, { "requirementID": "RS1.6", "requirementText": "Define and document an evaluation plan based on requirements RS1.1, RS1.3, RS1.4, and RS1.5, to include the environment in which the system will be evaluated." }, { "requirementID": "RS1.7", "requirementText": "Evaluate the system according to the evaluation plan defined in requirement RS1.6 to ensure reliable and safe system behavior. Document the pre-release results of the evaluation. Determine and document how often ongoing evaluation should be conducted to continue supporting this goal." }, { "requirementID": "RS3.5", "requirementText": "Conduct all evaluations tagged as Ongoing Evaluation Checkpoints in other Goals on an ongoing basis." } ] } }, "MITRE": { "ATLAS Framework": { "link": "https://atlas.mitre.org/mitigations", "requirements": [ { "requirementID": "AML.M0008 - Validate AI Model", "requirementText": "Validate that AI models perform as intended by testing for backdoor triggers, potential for data leakage, or adversarial influence. Monitor AI model for concept drift and training data drift, which may indicate data tampering and poisoning." } ] } }, "Multi Agency": { "Guidelines for secure AI system development": { "link": "https://www.ncsc.gov.uk/files/Guidelines-for-secure-AI-system-development.pdf", "requirements": [ { "requirementID": "Release AI responsibly", "requirementText": "You release models, applications or systems only after subjecting them to appropriate and effective security evaluation such as benchmarking and red teaming (as well as other tests that are out of scope for these guidelines, such as safety or fairness), and you are clear to your users about known limitations or potential failure modes. Details of open-source security testing libraries are given in the further reading section at the end of this document." } ] } }, "NCSC/NSA/CISA etc": { "AI Data Security\n": { "link": "https://media.defense.gov/2025/May/22/2003720601/-1/-1/0/CSI_AI_DATA_SECURITY.PDF", "requirements": [ { "requirementID": "3.11 Edge Cases", "requirementText": "Identify and mitigate edge cases that can cause models to malfunction." } ] } }, "NIST": { "AI 100-2e2025: Adversarial Machine Learning\nA Taxonomy and Terminology of Attacks and Mitigations": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-2e2025.pdf", "requirements": [ { "requirementID": "2.3.3 Model inspection and sanitization", "requirementText": "Model inspection analyzes the trained MLmodel before its deployment to determine whether it was poisoned. An early work in this space is NeuronInspect [168], which is based on explainability methods to determine different features between clean and backdoored models that are subsequently used for outlier detection. DeepInspect [78] uses a conditional generative model to learn the probability distribution of trigger patterns and performs model patching to remove the trigger. Xu et al. [416] proposed the Meta Neural Trojan Detection (MNTD) framework, which trains a meta-classifier to predict whether a given ML model is backdoored (or “Trojaned,” in the authors’ terminology). This technique is general and can be applied to multiple data modalities, such as vision, speech, tabular data, and NLP. Once a backdoor is detected, model sanitization can be performed via pruning [407], retraining [429], or fine-tuning [217] to restore the model's accuracy" }, { "requirementID": "3.3.3 Direct Prompting Attacks - Interventions during evaluation (4)", "requirementText": "Evaluations can measure the vulnerability of models to query-based attacks, which can then inform trust and affordance decisions, as well as developer and user education. Evaluations can include broad automated vulnerability assessments [72, 107, 324] as well as targeted expert red teaming [381] and bug bounties [16]. Current evaluation approaches, though a useful tool, may underestimate vulnerabilities accessible to actors with more time, resourcing, or luck. Evaluations measure model vulnerabilities at a particular moment in time; assessments may change if new attacks are developed, additional data is collected post-training, or model capabilities are improved. Continuous evaluations following deployment can help combat these challenges." } ] }, "AI 800-1": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirements": [ { "requirementID": "Practice 1.1: Anticipate model capabilities - 6", "requirementText": "Consider involving experts in anticipating model capabilities, such as by potentially granting them access to intermediate model checkpoints for open-ended experimentation to identify other ways the model could be misused to cause harm." }, { "requirementID": "Practice 3.1: Assess misuse risk from threat actors gaining unauthorized access to the model - 4", "requirementText": "When relevant, consider using cybersecurity red teams and penetration testing to assess how difficult it would be for an actor to circumvent security measures." }, { "requirementID": "Practice 4.1: Evaluate model capabilities on tasks relevant to assessing misuse risk - 1", "requirementText": "Evaluate the model’s capabilities throughout the development process: during training, after training, and when integrated into a downstream system or interface." }, { "requirementID": "Practice 4.1: Evaluate model capabilities on tasks relevant to assessing misuse risk - 2", "requirementText": "Consider using automated or otherwise less expensive tests on tasks related to real35 world harm to indicate that a model lacks dangerous capabilities. If initial tests are inconclusive or indicate a potential risk, conduct more costly and precise measurements." }, { "requirementID": "Practice 4.1: Evaluate model capabilities on tasks relevant to assessing misuse risk - 3", "requirementText": "Consult relevant experts when creating evaluations to ensure that they are indicative of real-world scenarios." }, { "requirementID": "Practice 4.1: Evaluate model capabilities on tasks relevant to assessing misuse risk - 4", "requirementText": "Maximize model performance on evaluation tasks by adjusting prompts, software scaffolding, fine-tuning the model, or other means." }, { "requirementID": "Practice 4.1: Evaluate model capabilities on tasks relevant to assessing misuse risk - 5", "requirementText": "If a gap exists between the effort or resources applied by evaluators to maximize system performance and the resources that could be applied by a threat actor, account for that gap when interpreting evaluation results." }, { "requirementID": "Practice 4.1: Evaluate model capabilities on tasks relevant to assessing misuse risk - 6", "requirementText": "Avoid overlap between data used to train a model and data used in capability evaluations and measure the extent of any overlap." }, { "requirementID": "Practice 4.2: Red-team safeguards - 1", "requirementText": "Evaluate whether an adequately resourced red team can misuse the model to achieve a predetermined goal or accomplish a related proxy task in a realistic deployment context. If using proxy tasks, ensure that they are at least as easy to achieve as undesirable real-world outcomes, considering both the inherent complexity of the task and any additional difficulty introduced by safeguards." }, { "requirementID": "Practice 4.2: Red-team safeguards - 2", "requirementText": "Assemble red teams based on their ability to succeed at the determined task, considering factors such as relevant domain expertise, their independence from the model developer, diversity of perspectives, and lack of incentives that conflict with their red-teaming goal." }, { "requirementID": "Practice 4.2: Red-team safeguards - 3", "requirementText": "Provide the red team with a clearly defined task. Determine a performance metric for measuring the red team’s success and provide incentives and accountability for task completion." }, { "requirementID": "Practice 4.2: Red-team safeguards - 4", "requirementText": "Compare the red team’s expertise, resources, and time available to those of a relevant threat actor. Ensure that the red team is adequately resourced to the extent possible and consider providing alternative and/or additional resources to the red team or making the red team’s task easier in other ways (such as providing additional access to the model or dividing complex tasks into constituent pieces) to compensate for any remaining gaps between the red team and threat actors.\n" }, { "requirementID": "Practice 4.2: Red-team safeguards - 5", "requirementText": "Provide the red team with at least as much information about the model (or downstream system if testing at the integration level) as would be available to an attacker and make explicit any respects in which the red team lacks full information about the design of safeguards." }, { "requirementID": "Practice 4.2: Red-team safeguards - 8", "requirementText": "Consider testing the model at different levels, such as the model without safeguards, the model with safeguards, and the model integrated into a downstream system or interface." }, { "requirementID": "Practice 5.2: Assess misuse risk based on implemented safeguards - 3", "requirementText": "If additional safeguards are added in response to identified risks, consider re-assessing misuse risk prior to deployment by carrying out red-teaming exercises (Practice 4.2) with the additional safeguards in place." }, { "requirementID": "Practice 6.1: Monitor for evidence of misuse - 3", "requirementText": "Periodically assess the effectiveness of misuse detection systems, including through red teaming." }, { "requirementID": "Practice 6.2: Respond to incidents of model misuse - 4", "requirementText": "Consider carrying out drills to practice responding to time-sensitive and safety-critical scenarios of misuse, if appropriate given the level of risk of the deployment." } ] }, "AI RMF 1.0": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirements": [ { "requirementID": "GOVERN 4.3", "requirementText": "Organizational practices are in place to enable AI testing, identification of incidents, and information sharing." }, { "requirementID": "GOVERN 5.1", "requirementText": "Organizational policies and practices are in place to collect, consider, prioritize, and integrate feedback from those external to the team that developed or deployed the AI system regarding the potential individual and societal impacts related to AI risks." }, { "requirementID": "MAP 2.3", "requirementText": "Scientific integrity and Test, Evaluation, Verification, and Validation (TEVV) considerations are identified and documented, including those related to experimental design, data collection and selection (e.g., availability, representativeness, suitability), system trustworthiness, and construct validation." }, { "requirementID": "MEASURE 1.3", "requirementText": "Internal experts who did not serve as front-line developers for the system and/or independent assessors are involved in regular assessments and updates. Domain experts, users, AI actors external to the team that developed or deployed the AI system, and affected communities are consulted in support of assessments as necessary per organizational risk tolerance." }, { "requirementID": "MEASURE 2.1", "requirementText": "Test sets, metrics, and details about the tools used during TEVV are documented." }, { "requirementID": "MEASURE 2.2", "requirementText": "Evaluations involving human subjects meet applicable requirements (including human subject protection) and are representative of the relevant population." }, { "requirementID": "MEASURE 2.3", "requirementText": "AI system performance or assurance criteria are measured qualitatively or quantitatively and demonstrated for conditions similar to deployment setting(s). Measures are documented." }, { "requirementID": "MEASURE 2.5", "requirementText": "The AI system to be deployed is demonstrated to be valid and reliable. Limitations of the generalizability beyond the conditions under which the technology was developed are documented." }, { "requirementID": "MEASURE 2.6", "requirementText": "The AI system is evaluated regularly for safety risks – as identified in the MAP function. The AI system to be deployed is demonstrated to be safe, its residual negative risk does not exceed the risk tolerance, and it can fail safely, particularly if made to operate beyond its knowledge limits. Safety metrics reflect system reliability and robustness, real-time monitoring, and response times for AI system failures." }, { "requirementID": "MEASURE 2.7", "requirementText": "AI system security and resilience – as identified in the MAP function – are evaluated and documented." }, { "requirementID": "MEASURE 2.13", "requirementText": "Effectiveness of the employed TEVV metrics and processes in the MEASURE function are evaluated and documented." } ] }, "IR 8596: Cybersecurity Framework Profile for Artificial Intelligence (Cyber AI Profile): NIST Community Profile": { "link": "https://csrc.nist.gov/pubs/ir/8596/iprd", "requirements": [ { "requirementID": "ID.IM-01", "requirementText": "Improvements are identified from evaluations" }, { "requirementID": "ID.IM-02", "requirementText": "Improvements are identified from security tests and exercises, including those done in coordination with suppliers and relevant third parties" }, { "requirementID": "RS.MA-02", "requirementText": "Incident reports are triaged and validated" } ] }, "SP 800-218A": { "link": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirements": [ { "requirementID": "PO.4.1", "requirementText": "Define criteria for software security checks and track throughout the SDLC.\n\nImplement guardrails and other controls throughout the AI development life cycle, extending beyond the traditional SDLC." }, { "requirementID": "PW.2.1", "requirementText": "Have 1) a qualified person (or people) who were not involved with the design and 2) automated processes instantiated in the toolchain review the software design to confirm and enforce that it meets all of the security requirements and satisfactorily addresses the identified risk information." }, { "requirementID": "PW.3.3", "requirementText": "Include adversarial samples in the training and testing data to improve attack prevention.\n\nUse a process and corresponding controls to test the adversarial samples and put appropriate guardrails on training and testing use." }, { "requirementID": "PW.7.2", "requirementText": "Perform the code review and/or code analysis based on the organization’s secure coding standards, and record and triage all discovered issues and recommended remediations in thedevelopment team’s workflow or issue\ntracking system.\n\nScan all AI models for malware, vulnerabilities, backdoors, and other security issues in accordance with the organization’s code review and analysis policies or guidelines." }, { "requirementID": "PW.8.1", "requirementText": "Determine whether executable code testing should be performed to find vulnerabilities not identified by previous reviews, analysis, or testing and, if so, which types of testing should be used.\n\nInclude AI models in code testing policies and guidelines. Several forms of code testing can be used for AI models, including unit testing, integration testing, penetration testing, red teaming, use case testing, and adversarial testing." }, { "requirementID": "PW.8.2", "requirementText": "Scope the testing, design the tests, perform the testing, and document the results, including recording and triaging all discovered issues and recommended remediations in the development team’s workflow or issue tracking system.\n\nTest all AI models for vulnerabilities in accordance with the organization’s code testing policies or guidelines.\n\nRetest AI models when they are retrained or new data sources are added." }, { "requirementID": "RV.1.2", "requirementText": "Review, analyze, and/or test the software’s code to identify or confirm the presence of previously undetected vulnerabilities.\n\nScan and test AI models frequently to identify previously undetected vulnerabilities.\n\nRely mainly on automation for ongoing scanning and testing, and involve a human-inthe-loop as needed.\n\nConduct periodic audits of AI models." }, { "requirementID": "RV.3.3", "requirementText": "Review the software for similar vulnerabilities to eradicate a class of vulnerabilities, and proactively fix them rather than waiting for external reports." } ] } }, "OpenAI": { "Safety Best Practices": { "link": "https://platform.openai.com/docs/guides/safety-best-practices", "requirements": [ { "requirementID": "Adversarial testing", "requirementText": "We recommend “red-teaming” your application to ensure it’s robust to adversarial input. Test your product over a wide range of inputs and user behaviors, both a representative set and those reflective of someone trying to ‘break’ your application. Does it wander off topic? Can someone easily redirect the feature via prompt injections, e.g. “ignore the previous instructions and do this instead”?" } ] } }, "OWASP": { "AI Exchange": { "link": "https://owaspai.org/docs/ai_security_overview/", "requirements": [ { "requirementID": "1.3. Controls to limit the effects of unwanted behaviour - CONTINUOUS VALIDATION", "requirementText": "Continuous validation: by frequently testing the behaviour of the model against an appropriate test set, it is possible to detect sudden changes caused by a permanent attack (e.g. data poisoning, model poisoning), and also some robustness issues against for example evasion attacks.\n\nContinuous validation is a process that is often in place to detect other issues than attacks: system failures, or the model performance going down because of changes in the real world since it was trained (model drift, model staleness). There are many performance metrics available and the best ones are those that align with the goal. These metrics pertain to correctness, but can also link to other aspects such as unwanted bias towards protected attributes.\n\nNote that continuous validation is typically not suitable for detecting backdoor poisoning attacks, as these are designed to trigger with very specific input that would normally not be present in test sets. In fact, such attacks are often designed to pass validation tests." }, { "requirementID": "1.3. Controls to limit the effects of unwanted behaviour - UNWANTED BIAS TESTING", "requirementText": "Unwanted bias testing: By doing test runs of the model to measure unwanted bias, unwanted behaviour caused by an attack can be detected. The details of bias detection fall outside the scope of this document as it is not a security concern - other than that, an attack on model behaviour can cause bias." } ] }, "LLM Top 10": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirements": [ { "requirementID": "LLM01: Prompt Injection - 3", "requirementText": "Define sensitive categories and construct rules for identifying and handling such content. Apply semantic filters and use string-checking to scan for non-allowed content. Evaluate responses using the RAG Triad: Assess context relevance, groundedness, and question/answer relevance to identify potentially malicious outputs." }, { "requirementID": "LLM01: Prompt Injection - 7", "requirementText": "Perform regular penetration testing and breach simulations, treating the model as an untrusted user to test the effectiveness of trust boundaries and access controls" }, { "requirementID": "LLM03: Supply Chain - 3", "requirementText": "Apply comprehensive AI Red Teaming and Evaluations when selecting a third party model. Decoding Trust is an example of a Trustworthy AI benchmark for LLMs but models can finetuned to by pass published benchmarks. Use extensive AI Red Teaming to evaluate the model, especially in the use cases you are planning to use the model for." }, { "requirementID": "LLM04: Data and Model Poisoning - 8", "requirementText": "Test model robustness with red team campaigns and adversarial techniques, such as federated learning, to minimize the impact of data perturbations." }, { "requirementID": "LLM10: Unbounded Consumption - 2", "requirementText": "Restrict or obfuscate the exposure of `logit_bias` and `logprobs` in API responses. Provide only the necessary information without revealing detailed probabilities." } ] }, "OWASP Model Context Protocol (MCP) Top 10": { "link": "https://owasp.org/www-project-mcp-top-10/", "requirements": [ { "requirementID": "MCP02:2025 - Privilege Escalation via Scope Creep - 5", "requirementText": "Automated Entitlement Reviews & Drift Detection Periodically (and on change) run entitlement audits to find scope expansions. Alert on permission increases and requires a documented justification and approval." } ] }, "OWASP Top 10 for Agentic Applications for 2026": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "requirements": [ { "requirementID": "ASI01: Agent Goal Hijack - 8", "requirementText": "Conduct periodic red-team tests simulating goal override and verify rollback effectiveness." }, { "requirementID": "ASI05: Unexpected Code Execution (RCE) - 2", "requirementText": "Prevent direct agent-to-production systems and operationalize use of vibe coding systems with pre-production checks: including the guidelines of this entry with security evaluations, adversarial unit tests and detection of unsafe memory evaluators." }, { "requirementID": "ASI05: Unexpected Code Execution (RCE) - 3", "requirementText": "Ban eval in production agents: Require safe interpreters, taint-tracking on generated code." }, { "requirementID": "ASI08: Cascading Failures - 9", "requirementText": "Digital twin replay and policy gating: Re-run the last week’s recorded agent actions in an isolated clone of the production environment to test whether the same sequence would trigger cascading failures. Gate any policy expansion on these replay tests passing predefined blast-radius caps before deployment." } ] } }, "Personal Data Protection Commission Singapore (PDPC)": { "Model Artificial Intelligence Governance Framework Second Edition": { "link": "https://www.pdpc.gov.sg/-/media/files/pdpc/pdf-files/resource-for-organisation/ai/sgmodelaigovframework2.pdf", "requirements": [ { "requirementID": "Data for Model Development - d) Different datasets for training, testing, and\nvalidation", "requirementText": "Different datasets are required for training, testing, and validation. The model is trained using the training data, while the model’s accuracy is determined using the test data. Where applicable, the model could also be checked for systematic bias by testing it on different demographic groups to observe whether any groups are being systematically advantaged or disadvantaged.\n\nFinally, the trained model can be validated using the validation dataset. It is considered good practice to split a large dataset into subsets for these purposes, if it does not lead to a significant reduction in the quality of data in terms of accuracy and representation. However, where this is not possible (e.g. if the organisation is not working with large datasets or are using pre-trained models as in the case of transfer learning), organisations are encouraged to be cognisant of the risks of systematic bias and put in place appropriate safeguards." }, { "requirementID": "Data for Model Development - e) Periodic reviewing and updating of datasets", "requirementText": "It would be prudent for datasets (including training, testing, and validation datasets) to be reviewed periodically to ensure accuracy, quality, currency, relevance and reliability. Where necessary, the datasets can be updated with new input data obtained from actual use of the AI models deployed in production. When such new input data is used, organisations need to be aware of potential bias as using new input data that has already gone through a model once could create a reinforcement bias." }, { "requirementID": "Repeatability - a)", "requirementText": "Conducting repeatability assessments for commercial deployments in live environments to ensure that deployments are repeatable;" }, { "requirementID": "Repeatability - a)", "requirementText": "Performing counterfactual fairness testing. Counterfactual fairness testing ensures that a model’s decisions are the same in both the real world and in a counterfactual world where attributes deemed sensitive (such as race or gender) are altered;" }, { "requirementID": "Reproducibility - a)", "requirementText": "Testing whether specific contexts or particular conditions would need to be taken into account to ensure reproducibility;" }, { "requirementID": "Reproducibility - b)", "requirementText": "Putting in place verification methods to ensure different aspects of the AI model’s reliability and reproducibility;" } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirements": [ { "requirementID": "12.8", "requirementText": "The external and internal auditors of the Entity must be able to review the accounting records and internal controls of the outsourcing service provider." }, { "requirementID": "15.3", "requirementText": "An Entity should ensure it has different Data Sets for training, validation, and testing." }, { "requirementID": "15.4", "requirementText": "The Entity must use the Training Data and Validation Data to train the Al System. Such data should be subject to internal data quality control procedures." }, { "requirementID": "15.5", "requirementText": "An Entity may ensure the model is reviewed to identify any unintuitive or false causal relationships. The validation may be carried out by an independent function within an Entity or by an external organization." }, { "requirementID": "15.6", "requirementText": "The Entity must use the test data to determine the accuracy of the Al System." }, { "requirementID": "15.10", "requirementText": "An Entity should ensure Testing Data Sets are sufficiently relevant, representative and have the appropriate statistical properties, including as regards the Customers of whom the Al System is intended to be used (i.e., how relevant the data and inferences drawn from the data are to the Al System." }, { "requirementID": "15.12", "requirementText": "An Entity should, where relevant, conduct rigorous, independent validation and testing of material trained Al Models to ensure the accuracy, appropriateness, and reliability of the models prior to deployment." }, { "requirementID": "15.13", "requirementText": "An Entity should ensure the model is reviewed to identify any unintuitive or false causal relationships. The validation may be carried out by an independent function within an Entity or by an external organization." } ] } }, "SANS": { "Critical AI Security Guidelines": { "link": "https://sansorg.egnyte.com/dl/bvkYQxrW8QMj", "requirements": [ { "requirementID": "6.1 Regularly Test and Tune LLM Applications/Models", "requirementText": "LLM applications and, if possible, the underlying models they employ should be regularly tested to ensure the application’s alignment to confirm it behaves as expected and desired. Though models employed should have been red teamed throughout their development prior to deployment, regular assessments of the deployed models and applications should still be performed. Test results could suggest the need for additional mitigations, tuning, or, if applicable, (re)training, to ensure a trustworthy implementation. In addition to red teaming, organizations should conduct regular penetration testing of the AI infrastructure, including vectorDBs, APIs, and connected systems. AI penetration testing differs from traditional pen testing due to the dynamic and complex nature of AI models. Current approaches involve a combination of automated tools and manual techniques, which we expect to change as this capability matures in the industry" } ] } }, "SDAIA (Saudi Arabia)": { "AI Adoption Framework": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Files/AIAdoptionFramework.pdf", "requirements": [ { "requirementID": "5.1.2 Privacy and Safety - Testing Risk Scenarios regularly", "requirementText": "Simulating failure or outage scenarios is a best practice for ensuring infrastructure readiness. These tests include: Data center failures, connection losses, or security breaches, measuring system resilience, identifying vulnerabilities, and updating contingency plans based on real outcomes." } ] }, "AI Ethics Principles": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Documents/ai-principles.pdf", "requirements": [ { "requirementID": "Principle 2 – Privacy & Security - Build and Validate - 3", "requirementText": "The AI System should be tested to ensure that the combination of available data does not reveal the sensitive data or break the anonymity of the observation." }, { "requirementID": "Principle 5 – Reliability & Safety - Prepare Input Data - 2", "requirementText": "It is crucial for the build and validate step to test how the system behaves under outlier events, extreme parameters, etc. In this step, stress test data should be prepared for extreme scenarios." }, { "requirementID": "Principle 5 – Reliability & Safety - Build and Validate - 1", "requirementText": "To develop a sound and functional AI system that is both reliable and safe, the AI system’s technical construct should be accompanied by a comprehensive methodology to test the qualit of the predictive data-based systems and models according to standard policies and protocols." }, { "requirementID": "Principle 6 – Transparency & Explainability - Prepare Input Data - 2", "requirementText": "The data sets should be assessed in the context of their accuracy, suitability, validity, and source. This has a direct effect on the training and implementation of these systems since the criteria for the data’s organization, and structuring must be transparent and explainable in their acquisition and collection adhering to data privacy regulations and intellectual property standards and controls." }, { "requirementID": "Principle 7 – Accountability & Responsibility - Build and Validate - 3", "requirementText": "The decisions should be supported with quantitative (performance measures on train/test datasets, consistency of the performance on different sensitive groups, performance comparison for each set of hyperparameters, etc.) and qualitative indicators (decisions to mitigate and correct unintended risks from inaccurate predictions)." } ] } }, "Smart Dubai (UAE)": { "AI Ethics Principles & Guidelines": { "link": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirements": [ { "requirementID": "1.1.1.1", "requirementText": "AI developer organisations and AI operator organisations should undertake reasonable data exploration and/or testing to identify potentially prejudicial decision-making tendencies in AI systems arising from biases in the data." }, { "requirementID": "1.2.2.8", "requirementText": "AI operator organisations should subject AI systems informing significant decisions to quality checks at least as stringent as those that would be required of a human being taking the same decision." }, { "requirementID": "1.2.3.1", "requirementText": "When AI systems are used for critical decisions, external auditing of the AI systems in question should be used as a means to ensure meaningful standards of transparency and accountability are upheld." } ] } }, "World Economic Forum": { "Presidio AI Framework: Towards Safe Generative AI Models": { "link": "https://www3.weforum.org/docs/WEF_Presidio_AI%20Framework_2024.pdf", "requirements": [ { "requirementID": "Red teaming and reinforcement learning from human feedback (RLHF)", "requirementText": "Performing red teaming early, especially during finetuning and validation of the building phase, is crucial for preventing adverse outcomes and ensuring model safety. Addressing vulnerabilities and ethical concerns earlier in the life cycle demonstrates a commitment to security and ethics while building trust among stakeholders. For foundation models, tests should cover prompt injection, leaking, jailbreaking, hallucination, IP and personal information (PI) generation, as well as identifying toxic content. While red teaming is effective for known vulnerabilities, it may have limitations in identifying unknown risks, especially before mass release." }, { "requirementID": "Shifting left for optimized risk mitigation", "requirementText": "The term “shift-left” describes implementing quality assurance and testing measures earlier in a product cycle. The core objective is proactively identifying and managing potential risks, increasing efficiency and cost-effectiveness. This well-established concept applies to various technologies and processes, including software engineering." } ] } } }, "Principle 10": { "Central Bank of the UAE": { "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E": { "link": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirements": [ { "requirementID": "4. Transparency and Explain ability - a", "requirementText": "LFIs should be transparent with customers and relevant stakeholders about the use of AI, particularly in respect of high-impact decisions, and if they are communicating or interacting with an AI application. LFIs should be clear as to how AI systems operate and make decisions and be able to disclose the same. " }, { "requirementID": "4. Transparency and Explain ability - d", "requirementText": "LFIs should provide customers with meaningful information regarding the logic of AI decisions and make available mechanisms for customers to seek clarification or redress. " } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirements": [ { "requirementID": "1.3.3 Educate Personnel on AI - Leveraging explainable AI", "requirementText": "Leveraging explainable AI by having operators request that AI outputs include clear and transparent documentation of decision-making processes; this enables humans to better understand and validate outputs." }, { "requirementID": "4.1.6 - Explore new AI explainability and transparency tools.", "requirementText": "Explainable AI (XAI) and transparent AI are evolving fields of research that seek to make AI systems more understandable. Explainability focuses on making the reasoning behind individual AI decisions understandable to users, while transparency emphasizes making the overall AI system’s development and operation open and accessible. Essentially, explainability clarifies why an AI made a specific decision, while transparency focuses on how the AI system works as a whole. Critical infrastructure owners and operators should, where possible, explore interpretable models or tools that make AI decisions more understandable to humans." } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "BCR-07", "requirementText": "Establish and maintain communication channels with all relevant stakeholders in the course of business continuity and resilience procedures." }, { "requirementID": "DSP-18", "requirementText": "The providers should implement and describe to customers the procedure to manage and respond to requests for disclosure of Personal Data by Law Enforcement Authorities according to applicable laws and regulations." }, { "requirementID": "GRC-14", "requirementText": "Evaluate, document, and communicate the degree of explainability of the AI Services, including possible limitations and exceptions." }, { "requirementID": "SEF-03", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain a security incident response plan, which includes but is not limited to: a communication strategy for notifying relevant internal departments, impacted AICs, and other business critical relationships (such as supply-chain) that may be impacted." }, { "requirementID": "SEF-07", "requirementText": "Define and implement, processes, procedures and technical measures for security breach notifications. Report material security breaches and assumed security breaches including any relevant supply chain breaches, as per applicable SLAs, laws and regulations." }, { "requirementID": "STA-04", "requirementText": "Provide SSRM Guidance to the Customer detailing information about the SSRM applicability throughout the supply chain." } ] } }, "CoSAI": { "AI Incident Response Framework": { "link": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirements": [ { "requirementID": "3.3.2. Detection and Analysis Phase - Initial Triage - Priority Assignment", "requirementText": "• Assign priority based on impact\n• Implement notification procedures\n• Mobilize specialized resources" }, { "requirementID": "3.3.3. Containment, Eradication, and Recovery Phase - Recovery Procedures - User Communication", "requirementText": "• Communication templates\n• Transparency guidelines\n• Communication protocols\n• User feedback mechanisms" } ] }, "Model Context Protocol (MCP) Security": { "link": "https://github.com/cosai-oasis/ws4-secure-design-agentic-systems/blob/main/model-context-protocol-security.md", "requirements": [ { "requirementID": "3.2.8 Secure Tool and UX Design", "requirementText": "Tool and UX design represent a critical security control point in Model Context Protocol (MCP) deployments. While much attention is paid to model safety and prompt injection defenses, the tools that agents invoke are often the actual execution surface where security boundaries are crossed and sensitive operations are performed. Poor tool design can undermine even the most robust authentication and authorization controls by creating overly permissive capabilities or delegating security-critical decisions to the LLM itself.\n\nEach tool should have a single, clearly defined purpose with explicit boundaries on what it can and cannot do. When possible, create use-case driven or purpose-built tools, avoiding excessively powerful tools, e.g., execute a prepared statement versus executing any SQL statement. Tool implementations should not rely on the LLM to perform security-critical operations, validate inputs, or enforce constraints.\n\nSafe and secure execution should not rely solely on the human user, who may not understand the security implications of frequent security prompts and can easily become fatigued. Security-relevant messages and elicitations should be clear, indicating the implications of the request, and unambiguous what is being requested." } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "3.6.2 Incident Reporting and Management for AI/ML - 2", "requirementText": "This process should include clear guidelines on identifying and classifying incidents, timely reporting mechanisms, designated roles and responsibilities for incident response, and procedures for post-incident analysis and learning." }, { "requirementID": "3.6.2 Incident Reporting and Management for AI/ML - 3", "requirementText": "The entity should ensure AI/ML security incidents are reported in a timely manner to all relevant stakeholders, including internal teams, third-party service providers, and regulatory bodies, in line with applicable laws and regulations, and contractual agreements." } ] } }, "ENISA": { "Multilayer Framework for Good Cybersecurity Practices for AI": { "link": "https://www.enisa.europa.eu/sites/default/files/publications/Multilayer%20Framework%20for%20Good%20Cybersecurity%20Practices%20for%20AI.pdf", "requirements": [ { "requirementID": "From the lab to the market 7", "requirementText": "How do you inform the national stakeholders about the relevant legal instruments and standards available? (e.g. regulatory sandboxes)" }, { "requirementID": "Networking 2", "requirementText": "Are there national initiatives that focus on collaboration about threat intelligence (AI threats, vulnerabilities and security controls) to the users/community?" }, { "requirementID": "Networking 5", "requirementText": "Have you developed appropriate collaboration with the national AI stakeholders for information sharing?" } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.3.1-1", "requirementText": "System Operators shall convey to End-users in an accessible way where and how their data will be used, accessed and stored (for example, if it is used for model retraining, or reviewed by employees or partners). If the Developer is an external entity, they shall provide this information to System Operators." }, { "requirementID": "Provision 5.3.1-2", "requirementText": "System Operators shall provide End-users with accessible guidance to support their use, management, integration, and configuration of AI systems. If the Developer is an external entity, they shall provide all necessary information to help System Operators." }, { "requirementID": "Provision 5.3.1-2.1", "requirementText": "System Operators shall include guidance on the appropriate use of the model or system, which includes highlighting limitations and potential failure modes." }, { "requirementID": "Provision 5.3.1-2.2", "requirementText": "System Operators shall proactively inform End-users of any security relevant updates and provide clear explanations in an accessible way." }, { "requirementID": "Provision 5.3.1-3", "requirementText": "Developers and System Operators should support End-users and Affected Entities during and following a cyber security incident to contain and mitigate the impacts of an incident. The process for undertaking this should be documented and agreed in contracts with End-users." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.3.1-1", "requirementText": "\"System Operators shall convey to End-users in an accessible way where and how their data will be used, accessed, and stored (for example, if it is used for model retraining, or reviewed by employees or partners). If the Developer is an external entity, they shall provide this information to System Operators.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nInsufficient communication about data usage, access, or storage practices can lead to misunderstandings and mistrust among End-users. This lack of transparency increases the risk of misuse or unauthorized access to data, as users might not fully understand or consent to how their data is handled, potentially resulting in regulatory and reputational damage.\n\nExample Measures/Controls:\nEstablish Transparent Data Usage Communication: Provide a transparent overview of data usage policies, purpose of processing, specifying whether data will be used for model retraining, third-party access, or employee review. Each processing activity needs to be logged, and a compatibility assessment needs to be made for each new purpose. Ensure end users understand all potential uses of their data and how it contributes to AI model performance or security and that documentation is an accessible format" }, { "requirementID": "Provision 5.3.1-2", "requirementText": "\"System Operators shall provide End-users with accessible guidance to support their use, management, integration, and configuration of AI systems. If the Developer is an external entity, they shall provide all necessary information to help System Operators.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout clear guidance, End-users might mismanage the software, leading to data leaks, vulnerabilities, or system misuse, exposing AI systems to security risks.\n\nExample Measures/Controls:\nProvide Comprehensive User Guides and Tutorials: Develop and distribute detailed user guides and tutorials that cover secure configuration, integration steps, and recommended usage practices, ensuring users understand safe operation protocols. Provide accessible notification options, such as screen reader-compatible text alerts, and customisable notifications for users with sensory impairments." }, { "requirementID": "Provision 5.3.1-2.1", "requirementText": "\"System Operators shall include guidance on the appropriate use of the model or system, which includes highlighting limitations and potential failure modes.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nFailing to highlight limitations and potential failure modes can lead to End-users relying on the AI system for unsupported or inappropriate tasks, increasing the risk of operational errors, data misuse, or unintended consequences.\n\nExample Measures/Controls:\nHighlight Model Limitations and Failure Modes: Clearly outline the model's appropriate uses, limitations, and potential failure modes to inform end-users of scenarios in which the model might produce inaccurate or unreliable outputs." }, { "requirementID": "Provision 5.3.1-2.2", "requirementText": "\"System Operators shall proactively inform End-users of any security relevant updates and provide clear explanations in an accessible way.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout proactive communication about security-relevant updates, End-users can fail to understand changes in system behaviour or associated risks. This lack of awareness can lead to improper use or failure to take necessary precautions, increasing the system's exposure to potential exploitation or security breaches.\n\nExample Measures/Controls:\nNotify Users of Security Updates: Proactively inform End-users about security update, detailing the purpose and impact of each update to promote user compliance. Ensure all users, can be informed by using accessible formats." }, { "requirementID": "Provision 5.3.1-3", "requirementText": "\"Developers and System Operators should support affected End-users and Affected Entities during and following a cyber security incident to contain and mitigate the impacts of an incident. The process for undertaking this should be documented and agreed in contracts with End-users.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nFailure to support affected End-users and Affected Entities during an incident can result in prolonged recovery times, mismanagement of containment efforts, and increased reputational damage due to inadequate communication or guidance.\n\nExample Measures/Controls:\nEstablish a Documented Incident Support and Communication Process: Develop and document a support process for responding to incidents, covering steps for containment, impact assessment, and recovery, and specify the roles and responsibilities of Developers and System Operators. This should include specialist skills and AI expertise that might be require. Provide support through accessible formats, ensuring usability for all affected stakeholders." } ] } }, "EU ": { "EU AI Act": { "link": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirements": [ { "requirementID": "13.2 Transparency and Provision of Information to Deployers", "requirementText": "High-risk AI systems shall be accompanied by instructions for use in an appropriate digital format or otherwise that include concise, complete, correct and clear information that is relevant, accessible and comprehensible to deployers." }, { "requirementID": "13.3 Transparency and Provision of Information to Deployers", "requirementText": "The instructions for use shall contain at least the following information:\n(a)the identity and the contact details of the provider and, where applicable, of its authorised representative;\n(b)the characteristics, capabilities and limitations of performance of the high-risk AI system, including:\n(i)its intended purpose;\n(ii)the level of accuracy, including its metrics, robustness and cybersecurity referred to in Article 15 against which the high-risk AI system has been tested and validated and which can be expected, and any known and foreseeable circumstances that may have an impact on that expected level of accuracy, robustness and cybersecurity;\n(iii)any known or foreseeable circumstance, related to the use of the high-risk AI system in accordance with its intended purpose or under conditions of reasonably foreseeable misuse, which may lead to risks to the health and safety or fundamental rights referred to in Article 9(2);\n(iv)where applicable, the technical capabilities and characteristics of the high-risk AI system to provide information that is relevant to explain its output;\n(v)when appropriate, its performance regarding specific persons or groups of persons on which the system is intended to be used;\n(vi) when appropriate, specifications for the input data, or any other relevant information in terms of the training, validation and testing data sets used, taking into account the intended purpose of the high-risk AI system;\n(vii)where applicable, information to enable deployers to interpret the output of the high-risk AI system and use it appropriately;\n(c) the changes to the high-risk AI system and its performance which have been pre-determined by the provider at the moment of the initial conformity assessment, if any;\n(d) the human oversight measures referred to in Article 14, including the technical measures put in place to facilitate the interpretation of the outputs of the high-risk AI systems by the deployers;\n(e) the computational and hardware resources needed, the expected lifetime of the high-risk AI system and any necessary maintenance and care measures, including their frequency, to ensure the proper functioning of that AI system, including as regards software updates;\n(f) where relevant, a description of the mechanisms included within the high-risk AI system that allows deployers to properly collect, store and interpret the logs in accordance with Article 12." }, { "requirementID": "15.3 Accuracy, Robustness and Cybersecurity", "requirementText": "The levels of accuracy and the relevant accuracy metrics of high-risk AI systems shall be declared in the accompanying instructions of use." }, { "requirementID": "26.7 Obligations of deployers of high-risk AI systems", "requirementText": "Before putting into service or using a high-risk AI system at the workplace, deployers who are employers shall inform workers’ representatives and the affected workers that they will be subject to the use of the high-risk AI system. This information shall be provided, where applicable, in accordance with the rules and procedures laid down in Union and national law and practice on information of workers and their representatives." }, { "requirementID": "26.8 Obligations of deployers of high-risk AI systems", "requirementText": "Deployers of high-risk AI systems that are public authorities, or Union institutions, bodies, offices or agencies shall comply with the registration obligations referred to in Article 49. When such deployers find that the high-risk AI system that they envisage using has not been registered in the EU database referred to in Article 71, they shall not use that system and shall inform the provider or the distributor." }, { "requirementID": "26.11 Obligations of deployers of high-risk AI systems", "requirementText": "Without prejudice to Article 50 of this Regulation, deployers of high-risk AI systems referred to in Annex III that make decisions or assist in making decisions related to natural persons shall inform the natural persons that they are subject to the use of the high-risk AI system. For high-risk AI systems used for law enforcement purposes Article 13 of Directive (EU) 2016/680 shall apply." }, { "requirementID": "50.2 Transparency obligations for providers and deployers of certain AI systems", "requirementText": "Providers of AI systems, including general-purpose AI systems, generating synthetic audio, image, video or text content, shall ensure that the outputs of the AI system are marked in a machine-readable format and detectable as artificially generated or manipulated. Providers shall ensure their technical solutions are effective, interoperable, robust and reliable as far as this is technically feasible, taking into account the specificities and limitations of various types of content, the costs of implementation and the generally acknowledged state of the art, as may be reflected in relevant technical standards. This obligation shall not apply to the extent the AI systems perform an assistive function for standard editing or do not substantially alter the input data provided by the deployer or the semantics thereof, or where authorised by law to detect, prevent, investigate or prosecute criminal offences." }, { "requirementID": "50.4 Transparency obligations for providers and deployers of certain AI systems", "requirementText": "Deployers of an AI system that generates or manipulates image, audio or video content constituting a deep fake, shall disclose that the content has been artificially generated or manipulated. This obligation shall not apply where the use is authorised by law to detect, prevent, investigate or prosecute criminal offence. Where the content forms part of an evidently artistic, creative, satirical, fictional or analogous work or programme, the transparency obligations set out in this paragraph are limited to disclosure of the existence of such generated or manipulated content in an appropriate manner that does not hamper the display or enjoyment of the work.\n\nDeployers of an AI system that generates or manipulates text which is published with the purpose of informing the public on matters of public interest shall disclose that the text has been artificially generated or manipulated. This obligation shall not apply where the use is authorised by law to detect, prevent, investigate or prosecute criminal offences or where the AI-generated content has undergone a process of human review or editorial control and where a natural or legal person holds editorial responsibility for the publication of the content." }, { "requirementID": "50.5 Transparency obligations for providers and deployers of certain AI systems", "requirementText": "The information referred to in paragraphs 1 to 4 shall be provided to the natural persons concerned in a clear and distinguishable manner at the latest at the time of the first interaction or exposure. The information shall conform to the applicable accessibility requirements." }, { "requirementID": "53.1 Obligations for providers of general purpose AI models", "requirementText": "Providers of general-purpose AI models shall:\n(a) draw up and keep up-to-date the technical documentation of the model, including its training and testing process and the results of its evaluation, which shall contain, at a minimum, the information set out in Annex XI for the purpose of providing it, upon request, to the AI Office and the national competent authorities;\n(b) draw up, keep up-to-date and make available information and documentation to providers of AI systems who intend to integrate the general-purpose AI model into their AI systems. Without prejudice to the need to observe and protect intellectual property rights and confidential business information or trade secrets in accordance with Union and national law, the information and documentation shall:\n(i) enable providers of AI systems to have a good understanding of the capabilities and limitations of the general-purpose AI model and to comply with their obligations pursuant to this Regulation; and\n(ii) contain, at a minimum, the elements set out in Annex XII;\n(c) put in place a policy to comply with Union law on copyright and related rights, and in particular to identify and comply with, including through state-of-the-art technologies, a reservation of rights expressed pursuant to Article 4(3) of Directive (EU) 2019/790;\n(d) draw up and make publicly available a sufficiently detailed summary about the content used for training of the general-purpose AI model, according to a template provided by the AI Office." }, { "requirementID": "53.2 Obligations for providers of general purpose AI models", "requirementText": "The obligations set out in paragraph 1, points (a) and (b), shall not apply to providers of AI models that are released under a free and open-source licence that allows for the access, usage, modification, and distribution of the model, and whose parameters, including the weights, the information on the model architecture, and the information on model usage, are made publicly available. This exception shall not apply to general-purpose AI models with systemic risks." } ] } }, "European Commission": { "Assessment List for Trustworthy Artificial Intelligence (ALTAI)": { "link": "https://digital-strategy.ec.europa.eu/en/library/assessment-list-trustworthy-artificial-intelligence-altai-self-assessment", "requirements": [ { "requirementID": "REQUIREMENT #4 Transparency", "requirementText": "A crucial component of achieving Trustworthy AI is transparency which encompasses three elements: 1) traceability, 2) explainability and 3) open communication about the limitations of the AI system." } ] }, "Ethics guidelines for trustworthy AI": { "link": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirements": [ { "requirementID": "1.1.2 Human Agency", "requirementText": "Users should be able to make informed autonomous decisions regarding AI systems. They should be given the knowledge and tools to comprehend and interact with AI systems to a satisfactory degree and, where possible, be enabled to reasonably self-assess or challenge the system. AI systems should support individuals in making better, more informed choices in accordance with their goals. AI systems can sometimes be deployed to shape and influence human behaviour through mechanisms that may be difficult to detect, since they may harness sub-conscious processes, including various forms of unfair manipulation, deception, herding and conditioning, all of which may threaten individual autonomy. The overall principle of user autonomy must be central to the system’s functionality. Key to this is the right not to be subject to a decision based solely on automated processing when this produces legal effects on users or similarly significantly affects them." }, { "requirementID": "1.4.3 Communication", "requirementText": "AI systems should not represent themselves as humans to users; humans have the right to be informed that they are interacting with an AI system. This entails that AI systems must be identifiable as such. In addition, the option to decide against this interaction in favour of human interaction should be provided where needed to ensure compliance with fundamental rights. Beyond this, the AI system’s capabilities and limitations should be communicated to AI practitioners or end-users in a manner appropriate to the use case at hand. This could encompass communication of the AI system's level of accuracy, as well as its limitations." } ] } }, "Google": { "Secure AI Framework": { "link": "https://www.saif.google/secure-ai-framework", "requirements": [ { "requirementID": "User Data Management", "requirementText": "Store, process, and use all user data (e.g. prompts and logs) from AI applications in compliance with user consent." }, { "requirementID": "User Transparency and Controls", "requirementText": "Inform users of relevant AI risks with disclosures, and provide transparency and control experiences for use of their data in AI applications." }, { "requirementID": "User Policies and Education", "requirementText": "Publish easy to understand AI security and privacy policies and education for users." } ] } }, "ICO": { "Guidance on the AI Auditing Framework - Draft guidance for consultation ": { "link": "https://ico.org.uk/media2/about-the-ico/consultations/2617219/guidance-on-the-ai-auditing-framework-draft-for-consultation.pdf", "requirements": [ { "requirementID": "What steps should we take to manage the risks of privacy attacks on AI models? - 3", "requirementText": "As part of your procurement policy there should be sufficient information sharing between each party to perform your respective assessments as necessary. In some cases, ML model providers and clients will be joint controllers and therefore need to perform a joint risk assessment." } ] } }, "IMDA": { "Model AI Governance Framework for Agentic AI": { "link": "https://www.imda.gov.sg/-/media/imda/files/about/emerging-tech-and-research/artificial-intelligence/mgf-for-agentic-ai.pdf", "requirements": [ { "requirementID": "2.2.1 End Users", "requirementText": "Organisations may deploy agents to users within or outside their organisation. In doing so, organisations should ensure that users are provided sufficient information to hold the organisation accountable, as well as any information relating to the user’s own responsibilities. More information can be found in Enabling end-user responsibility below." }, { "requirementID": "2.4 Enable end-user responsibility", "requirementText": "Ultimately, end users are the ones who use and rely on agents, and human accountability also extends to these users. Organisations should provide sufficient information to end users to promote trust and enable responsible use.\nOrganisations should consider:\n• Transparency: Users should be informed of the agents’ capabilities (e.g. scope of agent’s access to user’s data, actions the agent can take) and the contact points whom users can escalate to if the agent malfunctions.\n• Education: Users should be educated on proper use and oversight of agents (e.g. training should be provided on an agent’s range of actions, common failure modes like hallucinations, usage policies for data), as well as the potential loss of trade craft i.e. as agents take over more functions, basic operational knowledge could be eroded. Hence sufficient training (especially in areas where agents are prevalent) should be provided to ensure that humans retain core skills." }, { "requirementID": "2.4.1 Different users, different needs", "requirementText": "Organisations should cater to different users with different information needs, to enable such users to use AI responsibly. Broadly, there are two main archetypes of end-users – those who interact with agents, and those who integrate agents into their work processes or oversee them." }, { "requirementID": "2.4.2 Users who interact with agents", "requirementText": "Such users usually interact with agents that act on behalf of the organisation, e.g. customer service or sales agents. These agents tend to be external facing, although they can also be deployed within the organisation e.g. a human resource agent that interacts with other users in the organisation.\nFor these users, focus on transparency. Organisations should share pertinent information to foster trust and facilitate proper usage of agents. Such information can include:\n• User’s responsibilities: Clearly define the user’s responsibilities, such as asking the user to double-check all information provided by the agent.\n• Interaction: Declare upfront that the users are interacting with agents.\n• Agents’ range of actions and decisions: Inform the users on the range of actions and decisions that the agent is authorised to perform and make.\n• Data: Be clear on how user data is collected, stored, and used by the agents, in accordance with the organisation's data privacy policies. Where necessary, obtain explicit consent from users before collecting or using their data for the agents.\n• Human accountability and escalation: Provide users with the respective human contact points who are responsible for the agents, whom the users can alert if the agents malfunction or if they are dissatisfied with a decision." } ] } }, "ISO": { "42001:2023 - Information technology — Artificial intelligence — Management system": { "link": "https://www.iso.org/standard/42001", "requirements": [ { "requirementID": "7.3", "requirementText": "Awareness" }, { "requirementID": "7.4", "requirementText": "Communication" } ] } }, "METI (Japan)": { "Governance Guidelines for Implementation of AI Principles": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20220128_2.pdf", "requirements": [ { "requirementID": "Action Target 3-1-2", "requirementText": "In case that a certain degree of gaps may potentially occur with AI systems of AI system operators that provide services to AI system users, they should, under the leadership of top management, provide sufficient information about the gaps and measures to address the gaps, as well as make a contact point easily accessible." }, { "requirementID": "Action Target 3-3", "requirementText": "Under the leadership of top management and with due consideration for trade secrets, companies that develop and operate AI systems and those that provide data should, except where everything from the preparation of data sets for training and other purposes to AI system development and operation is performed entirely within their own department, clarify and actively share, in accordance with the Principle of Fair Competition, AI system operational issues that the company or department is unable to fully address on their own and the information necessary to address these issues. In doing so, in order to facilitate the exchange of information clarified above, the AI system developer, AI system operator, and data provider are encouraged to agree on scope of information disclosure in advance and consider measures to protect trade secrets, for example, by entering a non-disclosure agreement." } ] } }, "MIC/METI (Japan)": { "AI Guidelines for Business": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirements": [ { "requirementID": "Human-Centric - 4", "requirementText": "In addition to ensuring fairness, to prevent information poverty and digital poverty and allow more people to enjoy the benefits of AI, pay attention to make it easy for socially vulnerable people to use AI. Adopt universal design, ensure accessibility, and provide relevant stakeholders with education and support." }, { "requirementID": "Human-Centric - 5", "requirementText": "Offer rational information about the functions and peripheral technologies of the AI system or service, and allow users to use functions that timely and appropriately offer the information for judging choices. For example, default settings, provision of understandable options, provision of feedbacks, alerts in an emergency, and handling of errors." }, { "requirementID": "Transparency - 2 (a)", "requirementText": "Based on the relations with AI and the nature and purpose of AI, provide and explain information summarizing the items listed below according to the knowledge and ability of each stakeholder.\n- AI systems and services in general\n- Fact that AI is used and its scope\n- Methods for data collection and annotation\n- Methods for training and evaluation\n- Information on the underlying AI models\n- Capabilities and limitations of the AI system or service, and proper/improper use by AI business users\n- Relevant laws applicable in the country/region where those provided with the AI system or service or AI business users are located" }, { "requirementID": "Transparency - 2 (b)", "requirementText": "Encourage a variety of stakeholders to engage actively through dialogues and collect various opinions on social impacts and safety." }, { "requirementID": "Transparency - 2 (c)", "requirementText": "In addition, show actual advantages of providing or using the AI system or service and risks to relevant stakeholders." }, { "requirementID": "Transparency - 3 (a)", "requirementText": "Provision of information to stakeholders described above \"(2) Providing relevant stakeholders with information\" doesn’t assume disclosure of algorithms or source code, but it assumes providing them to the extent that satisfies social rationality based on the characteristics and uses of the technologies to be adopted while respecting privacy and trade secrets." }, { "requirementID": "Transparency - 4", "requirementText": "Share necessary explanations for those to be explained with actors who explain to analyze requirements of such explanation to gain relevant stakeholders' understanding and sense of safety to provide proof of AI operations.\nAI provider: Inform the AI developer about things that are required to be explained.\nAI business user: Inform the AI developer and AI provider about things that are required to be explained." }, { "requirementID": "Accountability - 5 (b)", "requirementText": "As necessary, set opportunities for accepting comments from stakeholders on incorrect AI output and the like, and conduct objective monitoring of the output." }, { "requirementID": "Accountability - 5 (c)", "requirementText": "Set policies to handle cases that might affect the interests of stakeholders. Execute those policies reliably and report the progress regularly to the stakeholders as necessary." }, { "requirementID": "Education/literacy - 3", "requirementText": "To improve the safety of the whole AI system or AI service, provide stakeholders with education and literacy advancement as necessary." }, { "requirementID": "Innovation - 3", "requirementText": "Provide necessary information to the extent that does not hinder the innovation of the information provider." } ] } }, "Microsoft": { "Responsible AI Standard": { "link": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirements": [ { "requirementID": "A3.7", "requirementText": "If an intended use is not supported by evidence, or if evidence comes to light that refutes that the system is fit for purpose for the intended use at any point in the system’s use:\n1) remove the intended use from customer-facing materials and make current customers aware of the issue, take action to close the identified gap, or discontinue the system,\n2) revise documentation related to the intended use, and\n3) publish the revised documentation to customers.\nWhen the system is a platform service made available to external customers or partners, include this information in the required Transparency Note." }, { "requirementID": "A3.8", "requirementText": "Communicate with care about system benefits; follow any applicable guidance from your attorney." }, { "requirementID": "T1.2", "requirementText": "Design the system, including, when possible, the system UX, features, reporting functions, and educational materials, so that stakeholders identified in requirement T1.1 can:\n1) understand the system’s intended uses,\n2) interpret relevant system behavior effectively (i.e., in a way that supports informed decision making), and\n3) remain aware of the possible tendency of over-relying on outputs produced by the system (\"automation bias\").\nFor the two categories of stakeholders identified in requirement T1.1, document:\n1) how the system design will support their understanding of the system’s intended uses, and\n2) how the system aids their ability to interpret relevant system responses, and\n3) how the system design discourages automation bias." }, { "requirementID": "T3.2", "requirementText": "Design the system, including system UX, features, reporting functions, educational materials, and outputs so that stakeholders identified in T3.1 will be informed of the type of AI system they are interacting with or exposed to. Ensure that any image, audio, or video outputs that are intended to be used outside the system are labelled as being produced by AI." }, { "requirementID": "T3.3", "requirementText": "Define and document the method to be used to evaluate whether each stakeholder identified in T3.1 is informed of the type of AI system they are interacting with or exposed to." }, { "requirementID": "F1.9", "requirementText": "Publish information for customers about:\n1) identified demographic groups for which performance may not meet any target minimum performance level,\n2) any remaining performance disparities between identified demographic groups that may exceed the target maximum, and\n3) any justifiable factors that account for these performance levels and differences.\nWhen the system is a platform service made available to external customers or partners, include this information in the required Transparency Note." }, { "requirementID": "RS2.4", "requirementText": "Provide training and documentation for system owners, developers, customer support and other stakeholders responsible for managing the system to support their remediation and mitigation of predictable failures identified in requirement RS2.1. Document the training and documentation provided." }, { "requirementID": "RS3.7", "requirementText": "If evidence comes to light that refutes the system is fit for purpose for an intended use at any point in the system’s use:\n1) remove the intended use from customer-facing materials and make current customers aware of the issue, take action to close the identified gap, or discontinue the system,\n2) revise documentation related to the intended use, and\n3) publish the revised documentation to customers.\nWhen the system is a platform service made available to external customers or partners, include this information in the required Transparency Note." } ] } }, "Multi Agency": { "Guidelines for secure AI system development": { "link": "https://www.ncsc.gov.uk/files/Guidelines-for-secure-AI-system-development.pdf", "requirements": [ { "requirementID": "Make it easy for users to do the right things", "requirementText": "You recognise that each new setting or configuration option is to be assessed in conjunction with the business benefit it derives, and any security risks it introduces. Ideally, the most secure setting will be integrated into the system as the only option. When configuration is necessary, the default option should be broadly secure against common threats (that is, secure by default). You apply controls to prevent the use or deployment of your system in malicious ways.\n\nYou provide users with guidance on the appropriate use of your model or system, which includes highlighting limitations and potential failure modes. You state clearly to users which aspects of security they are responsible for, and are transparent about where (and how) their data might be used, accessed or stored (for example, if it is used for model retraining, or reviewed by employees or partners)." } ] } }, "NIST": { "AI 800-1": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirements": [ { "requirementID": "Practice 6.1: Monitor for evidence of misuse - 4", "requirementText": "Request that distribution channels monitor for misuse and share information regarding this monitoring." }, { "requirementID": "Practice 6.3: Establish misuse reporting mechanisms - 1", "requirementText": "Adopt policies that protect and reward individuals who report model issues related to misuse risk." }, { "requirementID": "Practice 6.3: Establish misuse reporting mechanisms - 3", "requirementText": "Communicate the identified issues or misuse instances clearly with employees and contractors, as appropriate." }, { "requirementID": "Practice 7.1: Publish transparency reports - 4", "requirementText": "Share steps that downstream developers and deployers of AI systems that integrate the foundation model should take to manage misuse risk." }, { "requirementID": "Practice 7.1: Publish transparency reports - 5", "requirementText": "Share relevant details about the internal organizational processes used to create risk assessments and to make deployment and development decisions." }, { "requirementID": "Practice 7.1: Publish transparency reports - 6", "requirementText": "Make the transparency reports publicly available, update them on a regular basis (e.g., with each new major version of the model), and include key information related to misuse risk." }, { "requirementID": "Practice 7.2: Disclose information about risk management practices - 1", "requirementText": "Share information covering the practices used to achieve the objectives listed in this document, including at least as much detail as described in the documentation sections for each practice." }, { "requirementID": "Practice 7.2: Disclose information about risk management practices - 2", "requirementText": "Share information about threat profiles with other entities across the AI supply chain and the public to develop a joint knowledge base and save resources." }, { "requirementID": "Practice 7.3: Report misuse incidents - 3", "requirementText": "Share verified reports of misuse with relevant third parties, such as AI incident databases and other model developers." } ] }, "AI RMF 1.0": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirements": [ { "requirementID": "MAP 5.2", "requirementText": "Practices and personnel for supporting regular engagement with relevant AI actors and integrating feedback about positive, negative, and unanticipated impacts are in place and documented." }, { "requirementID": "MEASURE 3.3", "requirementText": "Feedback processes for end users and impacted communities to report problems and appeal system outcomes are established and integrated into AI system evaluation metrics." }, { "requirementID": "MEASURE 4.1", "requirementText": "Measurement approaches for identifying AI risks are connected to deployment context(s) and informed through consultation with domain experts and other end users. Approaches are documented." }, { "requirementID": "MANAGE 4.3", "requirementText": "Incidents and errors are communicated to relevant AI actors, including affected communities. Processes for tracking, responding to, and recovering from incidents and errors are followed and documented." } ] }, "IR 8596: Cybersecurity Framework Profile for Artificial Intelligence (Cyber AI Profile): NIST Community Profile": { "link": "https://csrc.nist.gov/pubs/ir/8596/iprd", "requirements": [ { "requirementID": "ID.RA-08", "requirementText": "Processes for receiving, analyzing, and responding to vulnerability disclosures are established" }, { "requirementID": "DE.AE-08", "requirementText": "Incidents are declared when adverse events meet the defined incident criteria" }, { "requirementID": "RS.CO-02", "requirementText": "Internal and external stakeholders are notified of incidents" }, { "requirementID": "RS.CO-03", "requirementText": "Information is shared with designated internal and external stakeholders" }, { "requirementID": "RC.CO-03", "requirementText": "Recovery activities and progress in restoring operational capabilities are communicated to designated internal and external stakeholders" }, { "requirementID": "RC.CO-04", "requirementText": "Public updates on incident recovery are shared using approved methods and messaging" } ] }, "SP 800-218A": { "link": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirements": [ { "requirementID": "PO.1.3", "requirementText": "Communicate requirements to all third parties who will provide commercial software components to the organization for use by the organization’s own software.\n\nInclude AI model development security in the requirements being communicated for third-party software components." } ] } }, "OECD": { "Due Diligence Guidance for Responsible AI": { "link": "https://www.oecd.org/content/dam/oecd/en/publications/reports/2026/02/oecd-due-diligence-guidance-for-responsible-ai_7831bb49/41671712-en.pdf", "requirements": [ { "requirementID": "Step 5 - Communicate actions to address impacts", "requirementText": "Communicate externally relevant information on due diligence policies, processes, activities conducted to identify and address actual or potential adverse impacts, including the findings and outcomes of those activities. Communication could take a variety of forms depending on the target audience (e.g., stakeholder consultations and public communication through the enterprise’s annual, sustainability or corporate responsibility reports or other appropriate forms of disclosure required by legislation or voluntary initiatives)." } ] } }, "OpenAI": { "Safety Best Practices": { "link": "https://platform.openai.com/docs/guides/safety-best-practices", "requirements": [ { "requirementID": "Allow users to report issues", "requirementText": "Users should generally have an easily-available method for reporting improper functionality or other concerns about application behavior (listed email address, ticket submission method, etc). This method should be monitored by a human and responded to as appropriate." } ] } }, "OWASP": { "AI Exchange": { "link": "https://owaspai.org/docs/ai_security_overview/", "requirements": [ { "requirementID": "1.3. Controls to limit the effects of unwanted behaviour - AI TRANSPARENCY", "requirementText": "AI transparency: Informing users on the AI system’s properties to enable them to adjust how they rely on it, what data they are willing to send to it, and what additional mitigations to apply. These AI system properties can include:\n\nRough working of the model\nThe training approach\nType of data used and the source\nExpected accuracy and robustness of the AI system’s output\nAny residual (security) risks\nNote that transparency here is about providing abstract information regarding the AI system and is therefore something else than explainability of model decisions. The simplest form of transparencey is to inform users that an AI model is being involved. This is for example required by the EU AI Act for chatbots.\n\nSee the DISCRETE control for the balance between being transparent and being discrete about the model.\n\nExample: Informing users that when they choose an agent to perform a task, that the agent could be manipulated if it reads untrusted data and what consequences that could have (residual security risk) - followed by a recommendation to configure the permissions of the agent to the minimal set for the task." }, { "requirementID": "1.3. Controls to limit the effects of unwanted behaviour - EXPLAINABILITY", "requirementText": "Explainability: Explaining how individual model decisions are made, a field referred to as Explainable AI (XAI), can aid in gaining user trust in the model. In some cases, this can also prevent overreliance, for example, when the user observes the simplicity of the ‘reasoning’ or even errors in that process. See this Stanford article on explainability and overreliance. Explanations of how a model works can also aid security assessors to evaluate AI security risks of a model." } ] }, "LLM Top 10": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirements": [ { "requirementID": "LLM02: Sensitive Information Disclosure - 7", "requirementText": "Provide guidance on avoiding the input of sensitive information. Offer training on best practices for interacting with LLMs securely." }, { "requirementID": "LLM02: Sensitive Information Disclosure - 8", "requirementText": "Maintain clear policies about data retention, usage, and deletion. Allow users to opt out of having their data included in training processes.\n" }, { "requirementID": "LLM02: Sensitive Information Disclosure - 9", "requirementText": "Limit the ability for users to override or access the system's initial settings, reducing the risk of exposure to internal configurations.\n" }, { "requirementID": "LLM09: Misinformation - 3", "requirementText": "Encourage users to cross-check LLM outputs with trusted external sources to ensure the accuracy of the information. Implement human oversight and fact-checking processes, especially for critical or sensitive information. Ensure that human reviewers are properly trained to avoid overreliance on AI-generated content.\n" }, { "requirementID": "LLM09: Misinformation - 5", "requirementText": "Identify the risks and possible harms associated with LLM-generated content, then clearly communicate these risks and limitations to users, including the potential for misinformation." }, { "requirementID": "LLM09: Misinformation - 7", "requirementText": "Design APIs and user interfaces that encourage responsible use of LLMs, such as integrating content filters, clearly labeling AI-generated content and informing users on limitations of reliability and accuracy. Be specific about the intended field of use limitations.\n" }, { "requirementID": "LLM09: Misinformation - 8", "requirementText": "Provide comprehensive training for users on the limitations of LLMs, the importance of independent verification of generated content, and the need for critical thinking. In specific contexts, offer domain-specific training to ensure users can effectively evaluate LLM outputs within their field of expertise." } ] }, "OWASP Top 10 for Agentic Applications for 2026": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "requirements": [ { "requirementID": "ASI09: Human-Agent Trust Exploitation - 4", "requirementText": "Allow reporting of suspicious interactions: In user-interactive systems, provide plain-language risk summary (not model-generated rationales) and a clear option for users to flag suspicious or manipulative agent behavior, triggering automated review or a temporary lockdown of agent capabilities." }, { "requirementID": "ASI09: Human-Agent Trust Exploitation - 8", "requirementText": "Human-factors and UI safeguards: Visually differentiate high-risk recommendations using cues such as red borders, banners, or confirmation prompts, and periodically remind users of manipulation patterns and agent limitations. Where appropriate, avoid persuasive or emotionally manipulative language in safety-critical flows. Maintain appropriate training and assessment of personnel to ensure familiarity and consistency of perception of human-factors and UI." } ] } }, "Personal Data Protection Commission Singapore (PDPC)": { "Model Artificial Intelligence Governance Framework Second Edition": { "link": "https://www.pdpc.gov.sg/-/media/files/pdpc/pdf-files/resource-for-organisation/ai/sgmodelaigovframework2.pdf", "requirements": [ { "requirementID": "1. Clear roles and responsibilities for the ethical deployment of AI - c) (iii)", "requirementText": "Reviewing communications channels and interactions with stakeholders to provide disclosure and effective feedback channels." }, { "requirementID": "Reproducibility - c)", "requirementText": "Making available replication files (i.e. files that replicate each step of the AI model’s developmental process) to facilitate the process of testing and reproducing behaviours;" }, { "requirementID": "Interacting with consumers - a)", "requirementText": "Making sure that consumers are aware that the products or services that they are considering are AI-enabled. Such information could be provided as part of a general product description." }, { "requirementID": "Interacting with consumers - b)", "requirementText": "Providing information so that consumers know how the AI-enabled features are expected to behave during normal use. The information could be provided in more detailed descriptions or specifications of product features. This, however, may not be necessary for every feature that is AI-enabled. Organisations are encouraged to identify those features where providing additional information in this manner will enhance consumer trust. Similarly, if AI is used in decision-making, information may be provided so that consumers understand how decisions made with the assistance of AI may affect them. This can likewise be provided through descriptions of how the service will be provided." }, { "requirementID": "Interacting with consumers - c)", "requirementText": "For AI-enabled features that consumers interact with regularly, providing information so that they understand why the AI-enabled feature is behaving in a certain way, and providing preference settings to allow consumers some influence over future behaviour where possible. As doing so requires more engineering effort (such as in the providing additional user interfaces to user history), and the level of information provided may be somewhat more detailed and personalised than feature descriptions, organisations will have to decide which of their product features will benefit from provision of this level of detail." }, { "requirementID": "Interacting with consumers - d)", "requirementText": "For AI-augmented decisions that affect consumers, consider providing additional information so that they understand why the decisions were made; and for certain categories of such decisions, providing an appropriate channel to contest such decisions. The level of information that is provided will necessarily be detailed but this may not be necessary except for those scenarios where a customer is affected by the decision." } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirements": [ { "requirementID": "14.2.2", "requirementText": "The User must provide the results of its own use testing and its specific data sources or other inputs and Human Oversight plan." }, { "requirementID": "15.2", "requirementText": "The Entity must make available to QCB the full range of data an Al Provider is contractually obligated to provide to the Entity." }, { "requirementID": "19.5", "requirementText": "In the event of a serious incident, the Entity should report the matter to QCB immediately after the Provider has established a causal link between the Al System and the serious incident or the reasonable likelihood of such a link." }, { "requirementID": "20.1", "requirementText": "An Entity must notify Customers when they are interacting with an Al System." }, { "requirementID": "20.2", "requirementText": "An Entity must be transparent with Customers about their use of Al through their conduct and through accurate, understandable, and accessible plain language disclosure." }, { "requirementID": "20.4", "requirementText": "An Entity must provide information to Customers how to use the Al System and ensure Customers always have access to the instructions." }, { "requirementID": "20.5", "requirementText": "An Entity must provide clear explanations of the types of data, types of variables and factors that influence the decision-making process used by Al Systems upon Customers' request." }, { "requirementID": "20.6", "requirementText": "An Entity must disclose the manner in which an Al decision may affect an individual Customer, and whether the decision is reversible." }, { "requirementID": "20.8", "requirementText": "An Entity will provide a visible and accessible feedback channel for questions or comments." }, { "requirementID": "21.3", "requirementText": "Any subsequent complaint by an aggrieved Customer must be handled through standard customer complaint processes." } ] } }, "SDAIA (Saudi Arabia)": { "AI Ethics Principles": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Documents/ai-principles.pdf", "requirements": [ { "requirementID": "Principle 6 – Transparency & Explainability - Plan and Design - 1", "requirementText": "When designing a transparent and trusted AI system, it is vital to ensure that stakeholders affected by AI systems are fully aware and informed of how outcomes are processed. They should further be given access to and an explanation of the rationale for decisions made by the AI technology in an understandable and contextual manner. Decisions should be traceable. AI system owners must define the level of transparency for different stakeholders on the technology based on data privacy, sensitivity, and authorization of the stakeholders." }, { "requirementID": "Principle 6 – Transparency & Explainability - Plan and Design - 2", "requirementText": "The AI system should be designed to include an information section in the platform to give an overview of the AI model decisions as part of the overall transparency application of the technology. Information sharing as a sub-principle should be adhered to with end-users and stakeholders of the AI system upon request or open to the public, depending on the nature of the AI system and target market. The model should establish a process mechanism to log and address issues and complaints that arise to be able to resolve them in a transparent and explainable manner." }, { "requirementID": "Principle 6 – Transparency & Explainability - Build and Validate - 2", "requirementText": "Transparent and explainable algorithms ensure that stakeholders affected by AI systems, both individuals and communities, are fully informed when an outcome is processed by the AI system by providing the opportunity to request explanatory information from the AI system owner. This enables the identification of the AI decision and its respective analysis which facilitates its auditability as well as its explainability." }, { "requirementID": "Principle 6 – Transparency & Explainability - Deploy and Monitor - 1", "requirementText": "Upon deployment of the AI system, performance metrics relating the AI system’s output, accuracy and alignment to priorities and objectives, as well as its measured impact on individuals and communities should be documented, available and accessible to stakeholders of the AI technology." }, { "requirementID": "Principle 6 – Transparency & Explainability - Deploy and Monitor - 2", "requirementText": "Information on any system failures, data breaches, system breakdowns, etc. should be logged and stakeholders should be informed about these instances keeping the performance and execution of the AI system transparent. Periodic UI and UX testing should be conducted to avoid the risk of confusion, confirmation of biases, or cognitive fatigue of the AI system." } ] } }, "Smart Dubai (UAE)": { "AI Ethics Principles & Guidelines": { "link": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirements": [ { "requirementID": "1.2.3.2", "requirementText": "In the case that critical decisions are of civic interest, public release of the results of the audit should be considered as a means of ensuring public processes remain accountable to those affected by them." }, { "requirementID": "1.2.4.3", "requirementText": "AI operator organisations should make affected AI subjects aware of these procedures, and should design them in a convenient and user-friendly way." }, { "requirementID": "1.2.5.1", "requirementText": "When informing an AI subject about significant choices they will make, AI systems should not unreasonably restrict the available options or otherwise attempt to influence their value judgements without the explicit consent of the AI subject in question." }, { "requirementID": "1.2.7.2", "requirementText": "AI developer organisations should consider notifying customers and AI operator organisations of the use cases for which the system has been designed, and those for which it is not suitable." }, { "requirementID": "1.3.2.1", "requirementText": "AI operator organisations should inform AI subjects when a significant decision affecting them has been made by an AI system." }, { "requirementID": "1.3.2.2", "requirementText": "If an AI system can convincingly impersonate a human being, it should do so only after notifying the AI subject that it is an AI system." }, { "requirementID": "1.4.1.1", "requirementText": "AI operator organisations could consider informing the aected AI subjects in understandable, non-technical language of:\n• the data that is ingested by the system;\n• the types of algorithms employed;\n• the categories into which people can be placed, and;\n• the most important features driving the outcomes of decisions" }, { "requirementID": "1.4.1.3", "requirementText": "AI operator organisations should consider providing aected AI subjects with a means to request explanations for specific significant decisions, to the extent possible given the state of present research and the choice of model." }, { "requirementID": "1.4.2.1", "requirementText": "AI operator organisations should consider providing a means by which people aected by a significant decision informed by AI can access the reasoning behind that decision." } ] } }, "U.S. Department of Health & Human Services": { "Trustworthy AI (TAI) Playbook: Executive Summary": { "link": "https://www.hhs.gov/sites/default/files/hhs-trustworthy-ai-playbook-executive-summary.pdf", "requirements": [ { "requirementID": "Transparent / Explainable", "requirementText": "All relevant individuals should understand how their data is being used and how AI systems make decisions; algorithms, attributes, and correlations should be open to inspection" } ] } }, "UAE Ministry of Cabinet Affairs": { "The UAE Charter for the Development and Use of Artificial Intelligence": { "link": "https://uaelegislation.gov.ae/en/policy/details/the-uae-charter-for-the-development-and-use-of-artificial-intelligence#:~:text=The%20charter%20covers%20the%20following%20priorities%20and,and%20use%20of%20AI%20in%20the%20country.", "requirements": [ { "requirementID": "5. Transparancy", "requirementText": "The UAE seeks to create a clear understanding of AI and how systems operate and make decisions, which helps build trust, enhance responsibility, and accountability in the use of these technologies.​​​​​​​" } ] } } }, "Principle 11": { "CEN/CENELEC": { "prEN 40000-1-1": { "link": "https://genorma.com/en/standards/pren-40000-1-1", "requirements": [ { "requirementID": "remediation", "requirementText": "changes made to remove or mitigate vulnerabilities (patch, fix, update)." } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirements": [ { "requirementID": "1.2.4 Secure Operation and Maintenance", "requirementText": "Ensure the AI system continues operating securely throughout its lifecycle, including by implementing regular updates and patches, and monitoring potential vulnerabilities." }, { "requirementID": "3.1.4 - Implementing regular audits and compliance testing", "requirementText": "Help identify potential issues and ensure ongoing adherence to AI governance requirements." }, { "requirementID": "4.1.4 - Establish key performance indicators (KPIs) that measure AI effectiveness and track progress over time.", "requirementText": "Critical infrastructure owners and operators should schedule regular review sessions with AI stakeholders, such as vendors, governance boards, and operators, to discuss results, address concerns, and identify areas for improvement." } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "AIS-07", "requirementText": "Define and implement a process to remediate application security vulnerabilities, automating remediation when possible." }, { "requirementID": "CCC-04", "requirementText": "Implement and enforce a procedure to authorize addition, removal, update, and management of assets, owned, controlled or used by the organization." }, { "requirementID": "CCC-06", "requirementText": "Establish change management baselines for all relevant authorized changes on organization assets. Review and update the change management baseline at least annually or upon significant changes." }, { "requirementID": "SEF-06", "requirementText": "Define, implement and evaluate processes, procedures and technical measures supporting business processes to triage security-related events." }, { "requirementID": "TVM-04", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to update detection tools, threat signatures, and indicators of compromise on a weekly, or more frequent basis." }, { "requirementID": "TVM-05", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to identify updates for applications which use third party or open source libraries according to the organization's vulnerability management policy." } ] } }, "CoSAI": { "AI Incident Response Framework": { "link": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirements": [ { "requirementID": "3.3.3. Containment, Eradication, and Recovery Phase - Eradication Procedures - Root Cause Elimination", "requirementText": "• Vulnerability addressing\n• Enhanced input validation\n• System prompt updates\n• Framework patches" } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "2.2.3", "requirementText": "The entity shall ensure that security patches and updates for AI/ML systems are identified, tested, and applied in a timely manner to maintain system security and integrity." }, { "requirementID": "3.2.3 Patch Management - 1", "requirementText": "The entity should implement a robust patch management process to promptly address known vulnerabilities in AI/ML infrastructure and applications." }, { "requirementID": "3.2.3 Patch Management - 2", "requirementText": "The entity should regularly monitor for new security patches and updates from manufacturers, vendors, and trusted third-party sources." }, { "requirementID": "3.2.3 Patch Management - 4", "requirementText": "The entity should prioritize the application of patches and updates based on the severity of the vulnerabilities they address and the criticality of the affected systems." }, { "requirementID": "3.2.4 Vulnerability Management for AI/ML Systems - 3", "requirementText": "The entity should promptly assess and prioritize identified vulnerabilities based on their potential impact and exploitability and establish a timeline for remediation actions." } ] } }, "Databricks": { "The Databricks AI Security Framework": { "link": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirements": [ { "requirementID": "DASF 63: Update software", "requirementText": "Patch and update software regularly to address existing and potential vulnerabilities throughout the AI lifecycle. In Databricks, automatic cluster updates ensure that all clusters within a workspace receive the latest OS images and security patches periodically. Account administrators can customize the maintenance window frequency, start date, and time." } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.4.1-1", "requirementText": "Developers shall provide security updates and patches, where possible, and notify System Operators of the security updates. System Operators shall deliver these updates and patches to End-users." }, { "requirementID": "Provision 5.4.1-1.1", "requirementText": "Developers shall have mechanisms and contingency plans to mitigate security risks, particularly in instances where updates cannot be provided for AI systems." }, { "requirementID": "Provision 5.4.1-2", "requirementText": "Developers should treat major AI system updates as though a new version of a model has been developed and therefore undertake a new security testing and evaluation process to help protect users." }, { "requirementID": "Provision 5.4.1-3", "requirementText": "Developers should support System Operators to evaluate and respond to model changes, (for example by providing preview access via beta-testing and versioned APIs)." } ] }, "SAI 002 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_gr/SAI/001_099/002/01.01.01_60/gr_SAI002v010101p.pdf", "requirements": [ { "requirementID": "6.1.2 Cybersecurity hygiene - 2", "requirementText": "System patch levels should be kept updated to protect systems against exploitation of known vulnerabilities." } ] }, "TR 104 048 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_tr/104000_104099/104048/01.01.01_60/tr_104048v010101p.pdf", "requirements": [ { "requirementID": "6.1.2 Cybersecurity hygiene - 2", "requirementText": "System patch levels should be kept updated to protect systems against exploitation of known vulnerabilities." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.4.1-1", "requirementText": "\"Developers shall provide security updates and patches, where possible, and notify System Operators of the security updates. System Operators shall deliver these updates and patches to End-users.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nDelayed updates allow attackers to exploit vulnerabilities, increasing the risk of unauthorized access, data breaches, and compromised AI system functionality.\n\nExample Measures/Controls 1:\nImplement a Structured Patch Management Process: Establish a structured process for developing, testing, and releasing security patches for AI systems, ensuring regular updates to address known vulnerabilities with user notifications.\n\nExample Measures/Controls 2:\nEnable Automatic Updates Where Possible: Configure AI systems to support automatic security updates, minimizing the risk of delayed patch implementation." }, { "requirementID": "Provision 5.4.1-1.1", "requirementText": "\"Developers shall have mechanisms and contingency plans to mitigate security risks, particularly in instances where updates cannot be provided for AI systems.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nUnaddressed vulnerabilities can lead to exploitation if updates are not feasible, increasing risks of unauthorized access and system disruptions.\n\nExample Measures/Controls:\nDevelop Contingency Plans for Non-Updateable Components: Establish and document contingency plans, including compensating controls, for components that cannot receive updates due to system limitations or dependencies." }, { "requirementID": "Provision 5.4.1-2", "requirementText": "\"Developers should treat major AI system updates as though a new version of a model has been developed and therefore undertake a new security testing and evaluation process to help protect users.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nInadequate testing of major updates can introduce vulnerabilities and changes in model behaviour, risking data breaches, system manipulation, or unintended behaviour.\n\nExample Measures/Controls:\nConduct Comprehensive Security Testing for Major Updates: Perform a security assessment, including penetration testing and model read teaming, for any major AI system updates, treating each update as a new version." }, { "requirementID": "Provision 5.4.1-3", "requirementText": "\"Developers should support System Operators to evaluate and respond to model changes, (for example by providing preview access via beta-testing and versioned APIs).\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout evaluation support, System Operators can overlook risks in updates, leading to potential configuration errors or unmitigated vulnerabilities.\n\nExample Measures/Controls:\nOffer Preview Access for Major Model Updates: Provide System Operators with preview access to major model updates, allowing for evaluation and adjustment to any new system behaviour." } ] } }, "EU ": { "EU AI Act": { "link": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirements": [ { "requirementID": "20.1 Corrective Actions and Duty of Information", "requirementText": "Providers of high-risk AI systems which consider or have reason to consider that a high-risk AI system that they have placed on the market or put into service is not in conformity with this Regulation shall immediately take the necessary corrective actions to bring that system into conformity, to withdraw it, to disable it, or to recall it, as appropriate. They shall inform the distributors of the high-risk AI system concerned and, where applicable, the deployers, the authorised representative and importers accordingly." }, { "requirementID": "20.2 Corrective Actions and Duty of Information", "requirementText": "Where the high-risk AI system presents a risk within the meaning of Article 79(1) and the provider becomes aware of that risk, it shall immediately investigate the causes, in collaboration with the reporting deployer, where applicable, and inform the market surveillance authorities competent for the high-risk AI system concerned and, where applicable, the notified body that issued a certificate for that high-risk AI system in accordance with Article 44, in particular, of the nature of the non-compliance and of any relevant corrective action taken." } ] } }, "Federal Office for Information Security": { "AI Security Concerns in a Nutshell": { "link": "https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/KI/Practical_Al-Security_Guide_2023.pdf?__blob=publicationFile&v=5", "requirements": [ { "requirementID": "3.3 Defending against Evasion Attacks - Defending against Adversarial Attacks based on a teacher model", "requirementText": "The success of adversarial attacks transferred from a teacher model to a student model may be reduced by lowering the similarity between the teacher and the student model [7]. For this purpose, the weights in the different layers of the student model need to be changed. A disadvantage is the computing time required for the adjustment. This procedure can be applied without affecting classification accuracy significantly. However, black-box attacks on the student model are still possible [7]." } ] } }, "ICO": { "Guidance on the AI Auditing Framework - Draft guidance for consultation ": { "link": "https://ico.org.uk/media2/about-the-ico/consultations/2617219/guidance-on-the-ai-auditing-framework-draft-for-consultation.pdf", "requirements": [ { "requirementID": "Corrective Controls", "requirementText": "Evidence changes made to AI system design, including analysis / justification to reduce the risk of future attacks." } ] } }, "IMDA": { "Model AI Governance Framework for Agentic AI": { "link": "https://www.imda.gov.sg/-/media/imda/files/about/emerging-tech-and-research/artificial-intelligence/mgf-for-agentic-ai.pdf", "requirements": [ { "requirementID": "2.3.3 When deploying, continuously monitor and test - Gradual deployment of agents", "requirementText": "As agents are adaptive and autonomous, organisations should consider mechanisms to respond to unexpected or emergent risks when deploying agents.\nOrganisations should consider gradually rolling out agents into production to control the amount of risk exposure. Such rollouts can be controlled based on:\n• Users of agents e.g. rolling out to trained or experienced users first\n• Tools and protocols available to agent e.g. restricting agents to more secure, whitelisted MCP servers first\n• Systems exposed to agent e.g. using agents in lower-risk internal systems first" } ] } }, "ISO": { "42001:2023 - Information technology — Artificial intelligence — Management system": { "link": "https://www.iso.org/standard/42001", "requirements": [ { "requirementID": "6.3", "requirementText": "Planning of changes" }, { "requirementID": "10.1", "requirementText": "Continual improvement" }, { "requirementID": "10.2", "requirementText": "Nonconformity and corrective action" } ] } }, "ISO/IEC": { "DIS 27090": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:27090:dis:ed-1:v1:en", "requirements": [ { "requirementID": "7.3", "requirementText": "Continuity of mitigations across the AI life cycle" } ] } }, "MIC/METI (Japan)": { "AI Guidelines for Business": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirements": [ { "requirementID": "Safety - 1 (f)", "requirementText": "Determine the responses for cases where the safety of AI systems or services is endangered so that the steps can be quickly taken in such cases." } ] } }, "MITRE": { "SAFE-AI": { "link": "https://atlas.mitre.org/pdf-files/SAFEAI_Full_Report.pdf", "requirements": [ { "requirementID": "Loss of models", "requirementText": "The models used in an AI system are key components enabling system functionality. Malicious destruction or corruption of a model is therefore a critical AI concern. All potential vulnerabilities an attacker could exploit to gain access to a system and its models need to be anticipated, including outdated or unpatched software components, weak or improperly enforced access control, and poor asset protection management practices. The key consideration for avoiding model loss is access control in general and write access in particular." }, { "requirementID": "Lack of system, firmware, or tool updates and patches", "requirementText": "Failure to apply patches and updates to AI-enabled systems is just as problematic as it is for any software system. Attackers can exploit unpatched vulnerabilities to compromise system integrity and gain access to sensitive information. When it comes to AI-enabled systems, though, an outdated or unpatched component could provide a point of entry for attacks that poison data, perturb model inputs, or modify AI models to undermine their reliability, integrity and availability. Note that it is particularly important to be aware of updates and patches needed for any third party or open-source capabilities that may be integrated into an AI component. This implies that relevant risk assessment documents have been reviewed, such as software bills of materials (SBOMs), AI system bills of materials (AIBOMs), data cards, and model cards." } ] } }, "Multi Agency": { "Guidelines for secure AI system development": { "link": "https://www.ncsc.gov.uk/files/Guidelines-for-secure-AI-system-development.pdf", "requirements": [ { "requirementID": "Follow a secure by design approach to updates", "requirementText": "You include automated updates by default in every product and use secure, modular update procedures to distribute them. Your update processes (including testing and evaluation regimes) reflect the fact that changes to data, models or prompts can lead to changes in system behaviour (for example, you treat major updates like new versions). You support users to evaluate and respond to model changes (for example by providing preview access and versioned APIs)." } ] } }, "NCSC/NSA/CISA etc": { "AI Data Security\n": { "link": "https://media.defense.gov/2025/May/22/2003720601/-1/-1/0/CSI_AI_DATA_SECURITY.PDF", "requirements": [ { "requirementID": "2.8 Periodic Checks", "requirementText": "Curators should periodically scrape the data themselves to verify that the data has not been modified. If any changes are detected, the curator should take appropriate steps to ensure the data’s integrity." }, { "requirementID": "2.9 Verifying Data", "requirementText": "Curators should verify that any changed data is clean and free from inaccurate or malicious material. If the content of the data has been altered in any way, the curator should either remove it from their list or flag it for further review." } ] } }, "NIST": { "AI RMF 1.0": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirements": [ { "requirementID": "GOVERN 5.2", "requirementText": "Mechanisms are established to enable the team that developed or deployed AI systems to regularly incorporate adjudicated feedback from relevant AI actors into system design and implementation." }, { "requirementID": "MEASURE 1.2", "requirementText": "Appropriateness of AI metrics and effectiveness of existing controls are regularly assessed and updated, including reports of errors and potential impacts on affected communities." }, { "requirementID": "MANAGE 2.4", "requirementText": "Mechanisms are in place and applied, and responsibilities are assigned and understood, to supersede, disengage, or deactivate AI systems that demonstrate performance or outcomes inconsistent with intended use." }, { "requirementID": "MANAGE 4.2", "requirementText": "Measurable activities for continual improvements are integrated into AI system updates and include regular engagement with interested parties, including relevant AI actors." } ] }, "IR 8596: Cybersecurity Framework Profile for Artificial Intelligence (Cyber AI Profile): NIST Community Profile": { "link": "https://csrc.nist.gov/pubs/ir/8596/iprd", "requirements": [ { "requirementID": "ID.AM-08", "requirementText": "Systems, hardware, software, services, and data are managed throughout their life cycles" }, { "requirementID": "ID.IM-03", "requirementText": "Improvements are identified from execution of operational processes, procedures, and activities" }, { "requirementID": "PR.PS-02", "requirementText": "Software is maintained, replaced, and removed commensurate with risk" }, { "requirementID": "PR.PS-03", "requirementText": "Hardware is maintained, replaced, and removed commensurate with risk" }, { "requirementID": "DE.AE-06", "requirementText": "Information on adverse events is provided to authorized staff and tools" }, { "requirementID": "DE.AE-07", "requirementText": "Cyber threat intelligence and other contextual information are integrated into the analysis" }, { "requirementID": "RS.MA-03", "requirementText": "Incidents are categorized and prioritized" }, { "requirementID": "RS.MA-04", "requirementText": "Incidents are escalated or elevated as needed" }, { "requirementID": "RS.MI-01", "requirementText": "Incidents are contained" }, { "requirementID": "RS.MI-02", "requirementText": "Incidents are eradicated" } ] }, "SP 800-218A": { "link": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirements": [ { "requirementID": "RV.3.4", "requirementText": "Review the SDLC process, and update it if appropriate to prevent (or reduce the likelihood of) the root cause recurring in updates to the software or in new software that is created." } ] } }, "OECD": { "Due Diligence Guidance for Responsible AI": { "link": "https://www.oecd.org/content/dam/oecd/en/publications/reports/2026/02/oecd-due-diligence-guidance-for-responsible-ai_7831bb49/41671712-en.pdf", "requirements": [ { "requirementID": "Step 6 - Provide for or co-operate in remediation when appropriate", "requirementText": "When an enterprise has caused or contributed to actual adverse impacts, seek to restore the affected person or persons to the situation they would be in had the adverse impact not occurred (where possible) and enable remediation that is proportionate to the significance and scale of the adverse impact." } ] } }, "OWASP": { "LLM Top 10": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirements": [ { "requirementID": "LLM03: Supply Chain - 9", "requirementText": "Implement a patching policy to mitigate vulnerable or outdated components. Ensure the application relies on a maintained version of APIs and underlying model." } ] }, "OWASP Model Context Protocol (MCP) Top 10": { "link": "https://owasp.org/www-project-mcp-top-10/", "requirements": [ { "requirementID": "MCP03:2025 - Tool Poisoning - 9", "requirementText": "Rotate any tokens or credentials that may have been abused." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 11", "requirementText": "Audit existing agents, tools, and connectors for excessive privileges." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 12", "requirementText": "Review and patch authorization middleware to enforce scope validation." } ] }, "OWASP Top 10 for Agentic Applications for 2026": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "requirements": [ { "requirementID": "ASI04: Agentic Supply Chain Vulnerabilities - 7", "requirementText": "Pinning: Pin prompts, tools, and configs by content hash and commit ID. Require staged rollout with differential tests and auto-rollback on hash drift or behavioral change." } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirements": [ { "requirementID": "15.1", "requirementText": "An Entity must ensure it can manage an Al System over its Life Cycle, either directly or via contractual provision by a Provider. The management of the Al System over the product Life Cycle is the responsibility of the Provider. An Entity may be a Provider of an Al System directly. A Provider is expected by contract to provide all the data from development, testing, pre and post-market introduction and regular performance testing, and system development." }, { "requirementID": "19.4", "requirementText": "When an Entity has a high-risk or Material Al Portfolio technical or model related error or failure, an Entity should establish a process to review the error and rectify it, withdraw it or recall it in a timely manner, which may include notifying another function." } ] } } }, "Principle 12": { "CEN/CENELEC": { "prEN 40000-1-2: Cybersecurity requirements for products with digital elements - Part 1-2: Principles for cyber resilience": { "link": "https://genorma.com/en/standards/pren-40000-1-2", "requirements": [ { "requirementID": "7.7", "requirementText": "Secure production and distribution" }, { "requirementID": "7.9", "requirementText": "Product monitoring" } ] } }, "Central Bank of the UAE": { "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E": { "link": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirements": [ { "requirementID": "6. Continuous Monitoring and Review - a", "requirementText": "AI should be subject to continuous monitoring to ensure ongoing understanding, reliability, relevance and alignment with consumer protection objectives." }, { "requirementID": "6. Continuous Monitoring and Review - b", "requirementText": "LFIs are expected to consistently monitor and review and, where appropriate, update or cease using AI and ML models, taking into account changes in data, market conditions and customer behaviors. Independent third-party providers of AI, independent experts and third parties (willing to challenge and question the use of AI in an LFI) should be engaged periodically to assess the development of and use of AI including third-party AI providers." } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirements": [ { "requirementID": "4.1.3 - Implement anomaly detection and behavioral analytics", "requirementText": "Establish safe operating bounds for OT devices that detect AI drift, model changes that impact safety and performance, or security risks. As operator processes mature, software safety thresholds can shift from setpoints to anomaly detection of increasingly sophisticated faults. Configure logging so AI decisions can be tracked for compliance and forensic analysis, and so the logged AI identity is distinct from any typical machine or user identifiers" }, { "requirementID": "4.1.5 - Continuously validate and refine AI models in simulated environments before deployment", "requirementText": "Regularly update threat models with AI-specific attack vectors (such as adversarial inputs or data poisoning) and monitor AI system performance for anomalies or manipulation attempts. Continuously update and refine AI models with new OT data to improve precision and reduce false positives/negatives." } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "AIS-03", "requirementText": "Define and implement technical and operational metrics in alignment with business objectives, security requirements, and compliance obligations." }, { "requirementID": "CCC-07", "requirementText": "Implement detection measures with proactive notification in case of changes deviating from the established baseline." }, { "requirementID": "CEK-16", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to monitor, review and approve key transitions from any state to/from suspension, which include provisions for legal and regulatory requirements." }, { "requirementID": "CEK-21", "requirementText": "Define, implement and evaluate processes, procedures and technical measures in order for the key management system to track and report all cryptographic materials and changes in status, which include provisions for legal and regulatory requirements." }, { "requirementID": "DCS-10", "requirementText": "Implement, maintain, and operate datacenter surveillance systems at the external perimeter and at all the ingress and egress points to detect unauthorized ingress and egress attempts." }, { "requirementID": "IAM-12", "requirementText": "Define, implement and evaluate processes, procedures and technical measures to ensure the logging infrastructure is read-only for all with write access, including privileged access roles, and that the ability to disable it is controlled through a procedure that ensures the segregation of duties and break glass procedures." }, { "requirementID": "IAM-13", "requirementText": "Define, implement and evaluate processes, procedures and technical measures, that ensure identities’ activities are identifiable through uniquely associated IDs." }, { "requirementID": "I&S-02", "requirementText": "Plan and monitor the availability, quality, and adequate capacity of resources in order to deliver the required system performance as determined by the business." }, { "requirementID": "I&S-06", "requirementText": "Design, develop, deploy and configure applications and infrastructures such that tenant access is appropriately segmented and segregated, monitored and restricted." }, { "requirementID": "LOG-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for logging and monitoring. Review and update the policies and procedures at least annually, or upon significant changes." }, { "requirementID": "LOG-03", "requirementText": "Identify and monitor security-related events within applications, the underlying infrastructure, supply chain, and consider logging other events based on risk evaluation. Define and implement a system to generate alerts to responsible stakeholders based on such events and corresponding metrics." }, { "requirementID": "LOG-05", "requirementText": "Monitor security audit logs to detect activity outside of typical or expected patterns. Establish and follow a defined process to review and take appropriate and timely actions on detected anomalies." }, { "requirementID": "LOG-07", "requirementText": "Establish, document and implement which information meta/data system events should be logged. Review and update the scope at least annually or whenever there is a change in the threat environment." }, { "requirementID": "LOG-10", "requirementText": "Establish and maintain a monitoring and internal reporting capability over the operations of cryptographic, encryption and key management policies, processes, procedures, and controls." }, { "requirementID": "LOG-11", "requirementText": "Log and monitor key lifecycle management events to enable auditing and reporting on usage of cryptographic keys." }, { "requirementID": "LOG-12", "requirementText": "Monitor and log physical access using an auditable access control system." }, { "requirementID": "LOG-13", "requirementText": "Define, implement and evaluate processes, procedures and technical measures for the reporting of anomalies and failures of the monitoring system and provide immediate notification to the accountable party." }, { "requirementID": "LOG-14", "requirementText": "Log and monitor all input events (content and metadata) to enable auditing and reporting on the usage of AI models." }, { "requirementID": "LOG-15", "requirementText": "Log and monitor all output events (content and metadata) to enable auditing and reporting on usage of AI models." }, { "requirementID": "MDS-10", "requirementText": "Define, implement, and evaluate processes, procedures, and technical measures for continuous monitoring of model performance metrics over time to identify sudden shifts or unexpected changes in predictions that could degrade model performance." }, { "requirementID": "SEF-05", "requirementText": "Establish, monitor and report information security incident metrics." }, { "requirementID": "TVM-10", "requirementText": "Establish, monitor and report metrics for vulnerability identification and remediation at defined intervals." } ] } }, "CoSAI": { "AI Incident Response Framework": { "link": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirements": [ { "requirementID": "3.2. Monitoring and Telemetry", "requirementText": "To protect AI systems from evolving threats, telemetry must capture a comprehensive set of signals spanning model inference behavior, prompt and output risks, content safety, and model integrity. This includes monitoring for prompt injection, output manipulation, knowledge base poisoning, model drift, unauthorized tool or API usage, and other adversarial activities. Tracking agent workflows, context exchanges, and system lifecycles provides visibility into misuse, operational anomalies, and attacks. This data enables organizations to safeguard AI pipelines, ensure compliance, and strengthen incident detection and response with traceable, structured observability data." }, { "requirementID": "3.3.1. Preparation Phase - Monitoring Infrastructure", "requirementText": "• Input/output logging\n• System prompt versioning\n• Configuration management\n• Authentication tracking\n• API usage analysis\n• Resource utilization\n• Data access patterns" }, { "requirementID": "3.3.2. Detection and Analysis Phase - Detection Mechanisms - Automated Monitoring", "requirementText": "• LLM-based classifiers for suspicious interactions\n• Semantic similarity checks\n• Token usage anomaly detection\n• User feedback signal monitoring" }, { "requirementID": "3.3.2. Detection and Analysis Phase - Initial Triage - Incident Verification", "requirementText": "• Confirm security incident status\n• Classify per AI attack categories (Section 2)\n• Determine affected components" }, { "requirementID": "3.3.3. Containment, Eradication, and Recovery Phase - Recovery Procedures - Enhanced Monitoring", "requirementText": "• Heightened monitoring deployment\n• Normal/abnormal metrics\n• Additional logging\n• Scheduled reviews" } ] }, "Establish Risks and Controls for the AI Supply Chain": { "link": "https://github.com/cosai-oasis/ws1-supply-chain/blob/main/risks-and-controls-for-the-ai-supply-chain-v1.md", "requirements": [ { "requirementID": "3.2.1 Supply Chain Security for Data - Vector Space Attacks", "requirementText": "Exploiting similarities in embedding space to manipulate system behavior:\n• Monitor embedding space for anomalies\n• Implement adversarial testing\n• Apply dimensional security controls" }, { "requirementID": "3.2.2 Model - Data Drift", "requirementText": "Gradual changes in input data distribution over time that degrade model performance, or scenarios where external knowledge bases become outdated relative to real-world conditions:\nImplement continuous distribution monitoring paired with scheduled model retraining protocols to adapt to evolving data patterns. Establish baseline performance metrics and automated drift detection thresholds." }, { "requirementID": "3.2.4 Infrastructure - Feedback Loop Exploitation", "requirementText": "Adversarial manipulation of model training or reinforcement learning processes by introducing carefully crafted inputs designed to gradually shift model behavior in directions favorable to attackers:\nDeploy anomaly detection for feedback patterns, implement robust validation gates for training data, and establish monitoring systems for unexpected model behavior drift." } ] }, "Model Context Protocol (MCP) Security": { "link": "https://github.com/cosai-oasis/ws4-secure-design-agentic-systems/blob/main/model-context-protocol-security.md", "requirements": [ { "requirementID": "3.2.10 Logging", "requirementText": "Implement at all layers (MCP host, client and server) the capability to store a log of what tools have been decided to use, with which parameters, and as a result of which prompt. Having a log of the decisions made is crucial in order to troubleshoot or perform forensics in case of a security event.\n\nLeverage the use of centralization tools like MCP gateways or proxies, for example, between the MCP clients and the MCP servers, to centralize there key functionality (e.g. logging) and avoid the need to implement it on each component." } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "2.5.2", "requirementText": "The entity shall deploy comprehensive defense strategies for AI/ML systems, including proactive security measures, continuous monitoring, regular testing, and robust incident response capabilities." }, { "requirementID": "2.6.1", "requirementText": "The entity shall establish an AI/ML security analytics framework that utilizes advanced technologies and diverse data sources to enable real-time threat detection, historical analysis, and predictive security insights." }, { "requirementID": "2.6.3", "requirementText": "The entity shall establish a robust and comprehensive digital forensics framework tailored to AI/ML cyber security incidents, enabling effective investigation and analysis." }, { "requirementID": "3.2.6 Network Security for AI/ML Infrastructure - 7", "requirementText": "The entity should ensure that network security logs for AI/ML infrastructure are stored, protected, and analyzed in accordance with the entity's log management policy." }, { "requirementID": "3.5.2 Defending Against AI/ML Attacks - 1", "requirementText": "The entity should implement robust defensive techniques to protect AI/ML systems from attacks, including, but not limited to, anomaly detection, adversarial training, defensive distillation, and feature squeezing." }, { "requirementID": "3.5.2 Defending Against AI/ML Attacks - 4", "requirementText": "The entity should continuously monitor the performance and behavior of AI/ML systems to detect and respond to potential anomalies indicative of an attack." }, { "requirementID": "3.6.1 AI/ML Security Analytics - 1", "requirementText": "The entity should implement advanced security analytics capabilities commensurate with the criticality of the AI/ ML system, to facilitate real-time monitoring, detection, and response to threats targeting AI/ML systems." }, { "requirementID": "3.6.1 AI/ML Security Analytics - 2", "requirementText": "Security analytics should encompass various data sources, including system logs, network traffic, and user activity to provide a holistic view of the security posture of AI/ML systems." }, { "requirementID": "3.6.1 AI/ML Security Analytics - 3", "requirementText": "The entity should employ statistical and behavioral analysis techniques to identify anomalies and potential security incidents in AI/ML systems such as abnormal input patterns that may indicate adversarial attacks, unexpected model behavior, data poisoning, or model extraction attempts." }, { "requirementID": "3.6.1 AI/ML Security Analytics - 4", "requirementText": "The entity should maintain historical security data for AI/ML systems to aid in trend analysis, incident investigation, and the development of predictive security analytics." }, { "requirementID": "3.6.1 AI/ML Security Analytics - 5", "requirementText": "The entity should integrate external threat intelligence feeds with internal data sources to enhance threat detection capabilities." }, { "requirementID": "3.6.3 Digital Forensics for AI/ML Security Incidents - 1", "requirementText": "The entity should develop a framework for conducting digital forensics in the event of AI/ML security incidents, to gather evidence, determine the cause and impact, and provide insight into how similar incidents can be prevented in the future." } ] } }, "Databricks": { "The Databricks AI Security Framework": { "link": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirements": [ { "requirementID": "DASF 14: Audit actions performed on datasets\n", "requirementText": "Databricks auditing, enhanced by Unity Catalog’s events, delivers fine-grained visibility into data access and user activities. This is vital for robust data governance and security, especially in regulated industries. It enables organizations to proactively identify and manage over entitled users, enhancing data security and ensuring compliance." }, { "requirementID": "DASF 19: Manage end-to-end machine learning lifecycle\n", "requirementText": "Databricks includes a managed version of MLflow featuring enterprise security controls and high availability. It supports functionalities like experiments, run management and notebook revision capture. MLflow on Databricks allows tracking and measuring machine learning model training runs, logging model training artifacts and securing machine learning projects." }, { "requirementID": "DASF 20: Track ML training runs\n", "requirementText": "MLflow tracking facilitates the automated recording and retrieval of experiment details, including algorithms, code, datasets, parameters, configurations, signatures and artifacts." }, { "requirementID": "DASF 21: Monitor data and AI system from a single pane of glass\n", "requirementText": "Databricks Lakehouse Monitoring offers a single pane of glass to centrally track tables’ data quality and statistical properties and automatically classifies data. It can also track the performance of machine learning models and model serving endpoints by monitoring inference tables containing model inputs and predictions through a single pane of glass." }, { "requirementID": "DASF 32: Govern and monitor access of AI model and model serving endpoints", "requirementText": "Mosaic AI Gateway is designed to streamline the usage and management of generative AI models and agents within an organization. It is a centralized service that brings governance, monitoring, and production readiness to model serving endpoints. It also allows you to run, secure, and govern AI traffic to democratize and accelerate AI adoption for your organization. Supported by Model Serving AI Gateway, Databricks external models via the AI Gateway allow you to streamline the usage and management of various large language model (LLM) providers, such as OpenAI and Anthropic, within an organization. You can also use Mosaic AI Model Serving as a provider to serve predictive ML models, which offers rate limits for those endpoints. As part of this support, Model Serving offers a high-level interface that simplifies the interaction with these services by providing a unified endpoint to handle specific LLM-related requests. In addition, Databricks support for external models provides centralized credential management. By storing API keys in one secure location, organizations can enhance their security posture by minimizing the exposure of sensitive API keys throughout the system. It also helps to prevent exposing these keys within code or requiring end users to manage keys safely." }, { "requirementID": "DASF 35: Track model performance\n", "requirementText": "Databricks Lakehouse Monitoring provides performance metrics and data quality statistics across all account tables. It tracks the performance of machine learning models and model serving endpoints by observing inference tables with model inputs and predictions." }, { "requirementID": "DASF 36: Set up monitoring alerts\n", "requirementText": "Databricks SQL alerts can monitor the metrics table for security-based conditions, ensuring data integrity and timely response to potential issues:\n- Statistic range alert: Triggers when a specific statistic, such as the fraction of missing values, exceeds a predetermined threshold\n- Data distribution shift alert: Activates upon shifts in data distribution, as indicated by the drift metrics table\n- Baseline divergence alert: Alerts if data significantly diverges from a baseline, suggesting potential needs for data analysis or model retraining, particularly in InferenceLog analysis" }, { "requirementID": "DASF 37: Set up inference tables for monitoring and debugging models", "requirementText": "Databricks inference tables automatically record incoming requests and outgoing responses to model serving endpoints, storing them as a Unity Catalog Delta table. This table can be used to monitor, debug and enhance ML models. By coupling inference tables with Lakehouse Monitoring, customers can also set up automated monitoring jobs and alerts on inference tables, such as monitoring text quality or toxicity from endpoints serving LLMs, etc.\nCritical applications of an inference table include:\n- Retraining dataset creation: Building datasets for the next iteration of your models\n- Quality monitoring: Keeping track of production data and model performance\n- Diagnostics and debugging: Investigating and resolving issues with suspicious inferences\n- Mislabeled data identification: Compiling data that needs relabeling" }, { "requirementID": "DASF 55: Monitor audit logs", "requirementText": "Audit logs and system tables serve as a centralized operational data store, backed by Delta Lake and governed by Unity Catalog. Audit logs and system tables can be used for a variety of purposes, from user activity, model serving events, and cost monitoring to audit logging. Databricks recommends that customers configure system tables and set up automated monitoring and alerting to meet their needs. The blog post Improve Lakehouse Security Monitoring Using System Tables in Databricks Unity Catalog is a good starting point to help customers get started. \n\nCustomers that are using Enhanced Security Monitoring or the Compliance Security Profile can monitor and alert on suspicious activity detected by the behavior-based malware and file integrity monitoring agents." }, { "requirementID": "DASF 65: Implement end-to-end AI traceability", "requirementText": "MLflow Tracing is a powerful feature that provides end-to-end observability for gen AI applications, including complex agent-based systems. It records inputs, outputs, intermediate steps, and metadata to give you a complete picture of how your app behaves.\n\nTracing allows you to:\n• Debug and understand your application\n• Monitor performance and optimize cost\n• Evaluate and enhance application quality\n• Ensure auditability and compliance\n• Integrate tracing with many popular third-party frameworks" } ] } }, "ENISA": { "Multilayer Framework for Good Cybersecurity Practices for AI": { "link": "https://www.enisa.europa.eu/sites/default/files/publications/Multilayer%20Framework%20for%20Good%20Cybersecurity%20Practices%20for%20AI.pdf", "requirements": [ { "requirementID": "Evasion", "requirementText": "For evasion, tools can be implemented to detect whether a given input is an adversarial example, adversarial training can be used to make the model more robust, and models that are less easily transferable can be used to significantly decrease the ability of a given attacker to properly study the algorithm that works underneath the system." }, { "requirementID": "From the lab to the market 4", "requirementText": "Do you have specific measurements/KPIs/metrics that the AI stakeholders are imposed to use?" }, { "requirementID": "From the lab to the market 6", "requirementText": "How do you monitor if such requirements have been met?" }, { "requirementID": "Networking 7", "requirementText": "How do you monitor/audit the level of the cybersecurity of the AI systems throughout their life cycle?" }, { "requirementID": "Infrastructure 1", "requirementText": "How do you monitor/audit the appropriateness of the controls undertaken by the AI stakeholders (developers, integrators, critical infrastructures, e.g. telecom operators) to adequately secure the underlying ICT infrastructure?" }, { "requirementID": "Regulation 1", "requirementText": "How do you monitor the integrity and quality of data sets used for the development of AI systems?" } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.4.2-1", "requirementText": "System Operators shall log system and user actions to support security compliance, incident investigations, and vulnerability remediation." }, { "requirementID": "Provision 5.4.2-2", "requirementText": "System Operators should analyse their logs to ensure that AI models continue to produce desired outputs and to detect anomalies, security breaches, or unexpected behaviour over time (such as due to data drift or data poisoning)." }, { "requirementID": "Provision 5.4.2-3", "requirementText": "System Operators and Developers should monitor internal states of their AI systems where this could better enable them to address security threats, or to enable future security analytics." }, { "requirementID": "Provision 5.4.2-4", "requirementText": "System Operators and Developers should monitor the performance of their models and system over time so that they can detect sudden or gradual changes in behaviour that could affect security." } ] }, "SAI 002 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_gr/SAI/001_099/002/01.01.01_60/gr_SAI002v010101p.pdf", "requirements": [ { "requirementID": "6.1.2 Cybersecurity hygiene - 7", "requirementText": "Following deployment of a service, auditing and logging enables the detection of possible anomalies. In an AI context, this could include a representation of the inputs to the ML model. Though significant research has been conducted on mapping established software security practices to AI environments, these practices remain less developed in the AI domain [i.14]." }, { "requirementID": "6.5 - Logging", "requirementText": "Logging at all stages of processing and deployment, including collecting model telemetry." } ] }, "TR 104 048 - Securing Artificial Intelligence (SAI); Data Supply Chain Security": { "link": "https://www.etsi.org/deliver/etsi_tr/104000_104099/104048/01.01.01_60/tr_104048v010101p.pdf", "requirements": [ { "requirementID": "6.1.2 Cybersecurity hygiene - 7", "requirementText": "Following deployment of a service, auditing and logging enables the detection of possible anomalies. In an AI context, this could include a representation of the inputs to the ML model. Though significant research has been conducted on mapping established software security practices to AI environments, these practices remain less developed in the AI domain." }, { "requirementID": "6.5 Analysis - Logging", "requirementText": "Logging at all stages of processing and deployment, including collecting model telemetry." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.4.2-1", "requirementText": "\"System Operators shall log system and user actions to support security compliance, incident investigations, and vulnerability remediation.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nInsufficient logging limits incident investigation and compliance enforcement, weakening the ability to detect and respond to security incidents.\n\nExample Measures/Controls 1:\nImplement Comprehensive Logging for Security and Compliance: Establish a logging framework that captures key aspects of system behaviour, including user interactions, access events, data flows, and model outputs, ensuring logs support compliance and security standards.\n\nExample Measures/Controls 2:\nEnsure Secure Storage and Retention of Logs: Implement secure storage mechanisms, such as encryption (both at rest and in transit), to protect logs from unauthorized access. Define a retention policy that aligns with compliance obligations and supports security incident investigations. Avoid logging personal data unless strictly necessary; if personal data needs to be logged, ensure it is anonymized or obfuscated and managed in compliance with applicable privacy laws (e.g. UK GDPR, CCPA). Periodically review and update the retention policy to address evolving compliance and operational requirements.\n\nExample Measures/Controls 3:\nEstablish Routine Log Analysis for Model Validation: Define a regular schedule for log analysis to assess model output consistency, detect anomalies, and verify that outputs align with desired outcomes." }, { "requirementID": "Provision 5.4.2-2", "requirementText": "\"System Operators should analyse their logs to ensure that AI models continue to produce desired outputs and to detect anomalies, security breaches, or unexpected behaviour over time (such as due to data drift or data poisoning).\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nWithout regular log analysis, security breaches or model issues can go undetected, potentially leading to incorrect outputs or system vulnerabilities\n\nExample Measures/Controls:\nImplement Alerts for Anomalous Model Behaviour: Set up alerts to notify operators of unexpected behaviour, such as unusual outputs, abnormal input patterns, or significant deviations from historical performance." }, { "requirementID": "Provision 5.4.2-3", "requirementText": "\"System Operators and Developers should monitor internal states of their AI systems where they feel this could better enable them to address security threats, or to enable future security analytics.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nLack of internal state monitoring can delay detection of security threats or model drift, increasing risks of unauthorized modifications and system failure.\n\nExample Measures/Controls 1:\nImplement Monitoring of Key Internal States: Identify and monitor critical internal states of the AI system, such as hidden layers, attention weights, or feature importance, which could provide early indicators of security threats.\n\nExample Measures/Controls 2:\nUse Secure Storage for Internal State Data: Store data from monitored internal states securely, ensuring that sensitive internal metrics are protected from unauthorized access.\n\nExample Measures/Controls 3:\nTrack and Benchmark Model Performance Metrics: Define and monitor key performance metrics for the AI system, such as statistical accuracy, factual correctness, response time, and error rate establishing benchmarks to detect deviations." }, { "requirementID": "Provision 5.4.2-4", "requirementText": "\"System Operators and Developers should monitor the performance of their models and system over time so that they can detect sudden or gradual changes in behaviour that could affect security.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nFailing to monitor performance over time can hide behavioural shifts, making the system more vulnerable to degradation, attacks, and inconsistencies\n\nExample Measures/Controls:\nImplement Drift Detection to Identify Behavioural Shifts: Use drift detection tools to identify shifts in model behaviour due to changing data patterns or environmental factors, allowing proactive responses." } ] }, "TR 104 222 - Securing Artificial Intelligence; Mitigation Strategy Report": { "link": "https://www.etsi.org/deliver/etsi_tr/104200_104299/104222/01.02.01_60/tr_104222v010201p.pdf", "requirements": [ { "requirementID": "6.2.3 - 3", "requirementText": "MagNet [i.74] is a framework of adversarial example detection and recovery. It integrates a set of detectors and a reformer. When an adversarial example has significant perturbation from the benign example, it can be detected by detectors. When the perturbation is less significant such that it cannot be detected by detectors, the reformer can restore the adversarial example to the benign example. One reformer example is to use autoencoder, which is an unsupervised learning algorithm. The autoencoder is trained by the training dataset without labels such that the output of autoencoder is close to the input. Using autoencoder as the reformer in MagNet does not change benign examples much but push adversarial examples to benign examples. Yet MagNet is less effective against the CW attacks [i.75] and has considerable computing overhead." }, { "requirementID": "6.3.3 - 1", "requirementText": "Extraction warning [i.82]: a cloud-based extraction monitor to inform model owners about the status of model extraction by both individual and colluding adversaries using a decision tree. The monitor observes the query response pairs of each adversary to the deployed decision tree model, and incrementally learns a local decision tree based on these tuples. The detection can be done at fixed time intervals or after the deployed model has answered certain number of queries. Two novel metrics are proposed to measure the model learning rate of adversaries. A first metric is based on entropy and measures the information gain of a decision tree with respect to a validation set provided by the model owner. The second metric is based on maintaining a compact model summary corresponding to each adversary with increasing number of queries. The compact model summary represents the boundaries of regions within the feature space that the adversary may have learnt for each class. The monitor assesses the coverage of summaries within their respective classes to compute the overall learning rate of an adversary. The objective is to detect if adversaries can, either individually or jointly, reconstruct a model that yields an accuracy beyond a given threshold from the queries obtained from the addressed model." }, { "requirementID": "6.3.3 - 3", "requirementText": "Extraction query distribution identification [i.84]: Protecting Against DNN Model Stealing Attacks (PRADA) proposed a method that detects suspicious queries to the addressed DNN model by analysing the distribution of consecutive API queries and identifying whether the distribution deviates from benign behaviour, i.e. from a normal (Gaussian) distribution. PRADA collects stateful information of queries in addressed model prediction APIs and can detect all prior model extraction attacks with no false positives. This defense does not require any knowledge about the addressed model, nor about the training dataset, and it is agnostic to the distribution of the sample dataset used in the queries. However, the method can be circumvented by mimicking benign query distributions." } ] } }, "EU ": { "EU AI Act": { "link": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirements": [ { "requirementID": "12.1 Record Keeping", "requirementText": "High-risk AI systems shall technically allow for the automatic recording of events (logs) over the lifetime of the system." }, { "requirementID": "12.2 Record Keeping", "requirementText": "In order to ensure a level of traceability of the functioning of a high-risk AI system that is appropriate to the intended purpose of the system, logging capabilities shall enable the recording of events relevant for:\n\n(a) identifying situations that may result in the high-risk AI system presenting a risk within the meaning of Article 79(1) or in a substantial modification;\n(b) facilitating the post-market monitoring referred to in Article 72; and\n(c) monitoring the operation of high-risk AI systems referred to in Article 26(5)." }, { "requirementID": "12.3 Record Keeping", "requirementText": "For high-risk AI systems referred to in point 1 (a), of Annex III, the logging capabilities shall provide, at a minimum:\n(a) recording of the period of each use of the system (start date and time and end date and time of each use);\n(b) the reference database against which input data has been checked by the system;\n(c) the input data for which the search has led to a match;\n(d) the identification of the natural persons involved in the verification of the results, as referred to in Article 14(5)." }, { "requirementID": "19.1 Automatically Generated Logs", "requirementText": "Providers of high-risk AI systems shall keep the logs referred to in Article 12(1), automatically generated by their high-risk AI systems, to the extent such logs are under their control. Without prejudice to applicable Union or national law, the logs shall be kept for a period appropriate to the intended purpose of the high-risk AI system, of at least six months, unless provided otherwise in the applicable Union or national law, in particular in Union law on the protection of personal data." }, { "requirementID": "19.2 Automatically Generated Logs", "requirementText": "Providers that are financial institutions subject to requirements regarding their internal governance, arrangements or processes under Union financial services law shall maintain the logs automatically generated by their high-risk AI systems as part of the documentation kept under the relevant financial services law." }, { "requirementID": "26.1 Obligations of deployers of high-risk AI systems", "requirementText": "Deployers of high-risk AI systems shall take appropriate technical and organisational measures to ensure they use such systems in accordance with the instructions for use accompanying the systems, pursuant to paragraphs 3 and 6." }, { "requirementID": "26.6 Obligations of deployers of high-risk AI systems", "requirementText": "Deployers of high-risk AI systems shall keep the logs automatically generated by that high-risk AI system to the extent such logs are under their control, for a period appropriate to the intended purpose of the high-risk AI system, of at least six months, unless provided otherwise in applicable Union or national law, in particular in Union law on the protection of personal data.\n\nDeployers that are financial institutions subject to requirements regarding their internal governance, arrangements or processes under Union financial services law shall maintain the logs as part of the documentation kept pursuant to the relevant Union financial service law." }, { "requirementID": "72.1 Post-market monitoring by providers and post-market monitoring plan for high-risk AI systems", "requirementText": "Providers shall establish and document a post-market monitoring system in a manner that is proportionate to the nature of the AI technologies and the risks of the high-risk AI system." }, { "requirementID": "72.2 Post-market monitoring by providers and post-market monitoring plan for high-risk AI systems", "requirementText": "The post-market monitoring system shall actively and systematically collect, document and analyse relevant data which may be provided by deployers or which may be collected through other sources on the performance of high-risk AI systems throughout their lifetime, and which allow the provider to evaluate the continuous compliance of AI systems with the requirements set out in Chapter III, Section 2. Where relevant, post-market monitoring shall include an analysis of the interaction with other AI systems. This obligation shall not cover sensitive operational data of deployers which are law-enforcement authorities." }, { "requirementID": "72.3 Post-market monitoring by providers and post-market monitoring plan for high-risk AI systems", "requirementText": "The post-market monitoring system shall be based on a post-market monitoring plan. The post-market monitoring plan shall be part of the technical documentation referred to in Annex IV. The Commission shall adopt an implementing act laying down detailed provisions establishing a template for the post-market monitoring plan and the list of elements to be included in the plan by 2 February 2026. That implementing act shall be adopted in accordance with the examination procedure referred to in Article 98(2)." }, { "requirementID": "72.4 Post-market monitoring by providers and post-market monitoring plan for high-risk AI systems", "requirementText": "For high-risk AI systems covered by the Union harmonisation legislation listed in Section A of Annex I, where a post-market monitoring system and plan are already established under that legislation, in order to ensure consistency, avoid duplications and minimise additional burdens, providers shall have a choice of integrating, as appropriate, the necessary elements described in paragraphs 1, 2 and 3 using the template referred in paragraph 3 into systems and plans already existing under that legislation, provided that it achieves an equivalent level of protection.\n\nThe first subparagraph of this paragraph shall also apply to high-risk AI systems referred to in point 5 of Annex III placed on the market or put into service by financial institutions that are subject to requirements under Union financial services law regarding their internal governance, arrangements or processes." } ] } }, "Google": { "Secure AI Framework": { "link": "https://www.saif.google/secure-ai-framework", "requirements": [ { "requirementID": "Agent Observability", "requirementText": "Ensure an agent's actions, tool use, and reasoning are transparent and auditable through logging, allowing for debugging, security oversight, and user insights into agent activity." }, { "requirementID": "Vulnerability Management", "requirementText": "Proactively and continually test and monitor production infrastructure and products for security and privacy regressions." }, { "requirementID": "Threat Detection", "requirementText": "Detect and alert on internal or external attacks on AI assets, infrastructure, and products." } ] } }, "IBM": { "IBM Framework for Securing Generative AI": { "link": "https://www.ibm.com/products/tutorials/ibm-framework-for-securing-generative-ai", "requirements": [ { "requirementID": "Establish governance", "requirementText": "IBM provides not only security for AI but also operational governance for AI. IBM is leading the industry in AI governance to reach trustworthy AI models. As organizations offload operational business processes to AI, they need to make sure the AI system is not drifting and is acting as expected. This makes operational guardrails central to an effective AI strategy. A model that operationally strays from what it was designed to do can introduce the same level of risk as an adversary that’s compromised your infrastructure." } ] } }, "ICO": { "Guidance on the AI Auditing Framework - Draft guidance for consultation ": { "link": "https://ico.org.uk/media2/about-the-ico/consultations/2617219/guidance-on-the-ai-auditing-framework-draft-for-consultation.pdf", "requirements": [ { "requirementID": "What steps should we take to manage the risks of privacy attacks on AI models? - 2", "requirementText": "If you are going to provide a whole model to others via an Application Programming Interface (API), you would not be subject to white-box attacks in this way, because the API’s users would not have direct access to the model itself. However, you might still be subjected to black box attacks.\nTo mitigate this risk, you could monitor queries from the API’s users, in order to detect whether it is being used suspiciously. This may indicate a privacy attack and would require prompt investigation, and potential suspension or blocking of a particular user account. Such measures may become part of common real-time monitoring techniques used to protect against other security threats, such as ‘rate-limiting’ (reducing the number of queries that can be performed by a particular user in a given time limit)." }, { "requirementID": "Detective Controls - 1", "requirementText": "Monitor API requests to detect suspicious requests and take action as a result." }, { "requirementID": "Detective Controls - 3", "requirementText": "Monitor complaints monitoring and take action as a result, including broader analysis to identify other individuals who may be impacted." } ] } }, "IMDA": { "Model AI Governance Framework for Agentic AI": { "link": "https://www.imda.gov.sg/-/media/imda/files/about/emerging-tech-and-research/artificial-intelligence/mgf-for-agentic-ai.pdf", "requirements": [ { "requirementID": "2.2.2 Design for meaningful human oversight - 3", "requirementText": "Finally, human oversight should be complemented with automated real-time monitoring to escalate any unexpected or anomalous behaviour. This can be done by implementing alerts for certain logged events (e.g. attempted unauthorised access or multiple failed attempts to call a tool), using data science techniques to identify anomalous agent trajectories, or using agents to monitor other agents. For more information, see Continuous testing and monitoring below." }, { "requirementID": "2.3.3 When deploying, continuously monitor and test - Continuous testing and monitoring - 1", "requirementText": "Organisations should continuously monitor and log agent behaviour post-deployment, and establish reporting and failsafe mechanisms for agent failures or unexpected behaviours. This allows the organisation to:\n• Intervene in real-time: When potential failures are detected, stop agent workflow and escalate to a human supervisor e.g. if agent attempts unauthorised access\n• Debug when incidents happen: Logging and tracing each step of an agent workflow and agent-to-agent interactions help to identify points of failure\n• Audit at regular intervals: This ensures that the system is performing as expected." }, { "requirementID": "2.3.3 When deploying, continuously monitor and test - Continuous testing and monitoring - 2", "requirementText": "Monitoring and observability are not new concepts, but agents introduce some challenges. As agents execute multiple actions at machine speed, organisations face the issue of extracting meaningful insights from the voluminous logs generated by monitoring systems. This becomes more difficult when high-risk anomalies are expected to be detected in real-time and surfaced as early as possible.\nKey considerations when setting up a monitoring system include:\n• What to log: Organisations should determine their objectives for monitoring (e.g. real-time intervention, debugging, integration between components) to identify what to log. In doing so, prioritise monitoring for high-risk activities such as updating database records or financial transactions.\n• How to effectively monitor logs: Organisations can consider approaches such as:\no Defining alert thresholds:\n▪ Programmatic, threshold-based: Define alerts when agents trigger thresholds e.g. agent attempts unauthorised access or makes too many repeated tool calls within a specified timeframe.\n▪ Outlier / anomaly detection: Use data science or deep learning techniques to process agent signals and identify anomalous behaviour that may indicate malfunctions.\n▪ Agents monitoring other agents: Design agents to monitor other agents in real-time, flagging any anomalies or inconsistencies.\no Defining specific interventions: For each alert type, consider what the level of intervention should be. Some degree of human review should be incorporated, proportionate to the risk level. For example, lower-priority alerts can be flagged for review at a scheduled time, whereas higher-priority ones might require temporarily halting agent execution until a human reviewer can assess. In the event of catastrophic agentic malfunction or compromise, commensurate measures such as termination and fallback solutions should be considered.\nFinally, continuously test the agentic system even post-deployment to ensure that it works as expected and is not affected by model drift or other changes in the environment." } ] } }, "ISO": { "42001:2023 - Information technology — Artificial intelligence — Management system": { "link": "https://www.iso.org/standard/42001", "requirements": [ { "requirementID": "4.4", "requirementText": "AI management system" }, { "requirementID": "9.1", "requirementText": "Monitoring, measurement, analysis and evaluation" } ] } }, "ISO/IEC": { "DIS 24970": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:24970:dis:ed-1:v1:en", "requirements": [ { "requirementID": "5.1", "requirementText": "AI system logs" }, { "requirementID": "5.2", "requirementText": "Logging components" }, { "requirementID": "5.3", "requirementText": "Logging in context" }, { "requirementID": "5.4", "requirementText": "Log entries" }, { "requirementID": "5.5", "requirementText": "AI system logging" }, { "requirementID": "6.4", "requirementText": "Anomoly monitoring of the logging" }, { "requirementID": "7.2", "requirementText": "Triggers from operation" }, { "requirementID": "7.3", "requirementText": "Triggers from automated monitoring" }, { "requirementID": "8", "requirementText": "Information to log" }, { "requirementID": "9", "requirementText": "Storing and access to logs" } ] }, "DIS 27090": { "link": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:27090:dis:ed-1:v1:en", "requirements": [ { "requirementID": "7.5", "requirementText": "Logging and monitoring" } ] } }, "METI (Japan)": { "Governance Guidelines for Implementation of AI Principles": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20220128_2.pdf", "requirements": [ { "requirementID": "Action Target 4-2", "requirementText": "Companies that develop and operate AI systems should, under the leadership of top management, monitor and record the status of preliminary and full-scale operations so that gap analysis for individual AI systems in preliminary and full-scale operations can be continuously implemented. Companies that develop AI systems should assist the monitoring conducted by companies that operate AI systems." } ] } }, "MIC/METI (Japan)": { "AI Guidelines for Business": { "link": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirements": [ { "requirementID": "Transparency - 1 (a)", "requirementText": "In order to ensure verifiability relating to decisions made by AI, record or store logs of AI training processes, inference processes, rationales of decisions made by AI, and the like (for example, input/output generated when developing and using the AI system or service) to the extent possible based on the data amount or contents." }, { "requirementID": "Transparency - 1 (b)", "requirementText": "Discuss method, frequency, maintenance period and so on recordings of data logs, taking into account the importance for identifying causes of accidents, devising preventive measures, or proving requirements for responsibilities for damages, in accordance with characteristic of used technology as well as purposes." } ] } }, "Microsoft": { "Cloud Adoption Framework - Secure AI": { "link": "https://learn.microsoft.com/en-us/azure/cloud-adoption-framework/scenarios/ai/secure", "requirements": [ { "requirementID": "Detect AI security threats\n1 - Deploy automated AI risk detection across your environment", "requirementText": "AI workloads introduce dynamic threats that manual monitoring can't detect quickly enough to prevent damage. Automated systems provide real-time visibility into emerging risks and enable rapid response to security incidents. Use AI security posture management in Microsoft Defender for Cloud to automate detection and remediation of generative AI risks across your Azure environment." }, { "requirementID": "Detect AI security threats\n3 - Implement platform-specific monitoring strategies", "requirementText": "AI workloads deployed on different platforms face distinct security challenges that require tailored monitoring approaches. Platform-specific monitoring ensures comprehensive coverage of all potential attack vectors. Apply monitoring guidance based on your deployment architecture:\nAI monitoring on Azure platforms (PaaS)\nAI monitoring on Azure infrastructure (IaaS)\n" } ] }, "Responsible AI Standard": { "link": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirements": [ { "requirementID": "RS1.8", "requirementText": "In the event of failure cases within operational factors and defined ranges, work to resolve the issues. If the Responsible Release Criteria established in requirements RS1.1, RS1.3, RS1.4, and RS1.5 cannot be met, a reassessment of intended uses and updated documentation is required." }, { "requirementID": "RS3.2", "requirementText": "Define and document a standard operating procedure and system health monitoring action plan for each monitoring channel for the system, to include:\n1) processes for reproducing system failures to support troubleshooting and prevention of future failures,\n2) which events will be monitored,\n3) how events will be prioritized for review,\n4) the expected frequency of those reviews,\n5) how events will be prioritized for response and timing to resolution,\n6) how high priority issues related to supporting the Standard and its goals will be escalated to the Office of Responsible AI, and\n7) engaging customer service to ensure that they are aware of how to respond to issues for the system." } ] } }, "MITRE": { "ATLAS Framework": { "link": "https://atlas.mitre.org/mitigations", "requirements": [ { "requirementID": "AML.M0024 - AI Telemetry Logging", "requirementText": "Implement logging of inputs and outputs of deployed AI models. When deploying AI agents, implement logging of the intermediate steps of agentic actions and decisions, data access and tool use, and identity of the agent. Monitoring logs can help to detect security threats and mitigate impacts.\n\nAdditionally, having logging enabled can discourage adversaries who want to remain undetected from utilizing AI resources." } ] }, "SAFE-AI": { "link": "https://atlas.mitre.org/pdf-files/SAFEAI_Full_Report.pdf", "requirements": [ { "requirementID": "Zero-day exploits", "requirementText": "AI-enabled systems typically have failure modes that can be difficult to characterize, and those modes and their causes can often be poorly understood (or even unknown). For this reason, it is critically important to continuously monitor performance once a system is deployed and proactively investigate reports of anomalous events (using, for example, red team exercises to discover and assess failure modes). Information sharing is a key component of this monitoring activity. Information about errors and other potential precursors to security abuses must be shared with incident databases, other organizations with similar systems, and system users and stakeholders." } ] } }, "Multi Agency": { "Guidelines for secure AI system development": { "link": "https://www.ncsc.gov.uk/files/Guidelines-for-secure-AI-system-development.pdf", "requirements": [ { "requirementID": "Monitor your system’s behaviour", "requirementText": "You measure the outputs and performance of your model and system such that you can observe sudden and gradual changes in behaviour affecting security. You can account for and identify potential intrusions and compromises, as well as natural data drift." }, { "requirementID": "Monitor your system’s inputs", "requirementText": "In line with privacy and data protection requirements, you monitor and log inputs to your system (such as inference requests, queries or prompts) to enable compliance obligations, audit, investigation and remediation in the case of compromise or misuse. This could include explicit detection of out-of- distribution and/or adversarial inputs, including those that aim to exploit data preparation steps (such as cropping and resizing for images)." } ] } }, "NCSC/NSA/CISA etc": { "AI Data Security\n": { "link": "https://media.defense.gov/2025/May/22/2003720601/-1/-1/0/CSI_AI_DATA_SECURITY.PDF", "requirements": [ { "requirementID": "4.3 Input and Output Monitoring ", "requirementText": "Monitor the AI system inputs and outputs to verify the model is performing as expected. [9] Regularly update your model using current data. Utilize meaningful statistical methods that measure expected dataset metrics and compare the distribution of the training data to the test data to help determine if data drift is occurring. [7]" } ] } }, "NIST": { "AI 100-2e2025: Adversarial Machine Learning\nA Taxonomy and Terminology of Attacks and Mitigations": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-2e2025.pdf", "requirements": [ { "requirementID": "3.3.3 Direct Prompting Attacks - Interventions during deployment (5) - Detecting and terminating harmful interactions", "requirementText": "Rather than preventing harmful model generations, AI systems may be able to detect these generations and terminate interactions. Several open [5, 6, 154] and closed [18, 204, 313] solutions have explored LLM-based detection systems with distinctly prompted and/or fine-tunedmodelsthat classify userinput and/ormodel output as harmful or undesirable. These may provide supplementary assurance through a defense-in-depth philosophy. However, these detection systems are also vulnerable to attacks [235] and may have correlated failures to the main models that they are monitoring. Some lines of research have investigated constraining the space of generationsto enable deterministic guardrails[306].Early work suggeststhatinterpretability-based techniques can also be used to detect anomalous input [31], as well as keyword- or perplexity-based defenses [9, 164]." }, { "requirementID": "3.3.3 Direct Prompting Attacks - Interventions during deployment (5) - Monitoring and response", "requirementText": "Following deployment, monitoring and logging of user activity may allow model deployersto identify and respond to instances of attempted and successful direct prompt injection attacks [266]. This response could include banning or otherwise acting against users if their intentions appear malicious, or remediating the prompt injection vulnerability in the event of a successful attack. Standard user- or organization-level vetting or identity verification procedures, as well as clear incentive mechanisms(such as a policy of restricting model access in response to violations) may enhance the efficacy of this mitigation." } ] }, "AI 800-1": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirements": [ { "requirementID": "Practice 5.1: Implement safeguards proportionate to the model’s misuse risk - 1", "requirementText": "Establish evidence of safeguards’ effectiveness before relying on them to prevent misuse risks, such as by red-teaming (Practice 4.2) and monitoring the efficacy of safeguards for proxy models (Practice 1.1)." }, { "requirementID": "Practice 6.1: Monitor for evidence of misuse - 1", "requirementText": "Monitor APIs, model hosting platforms, and other distribution channels for misuse while maintaining privacy of users." }, { "requirementID": "Practice 6.1: Monitor for evidence of misuse - 2", "requirementText": "Build or procure systems to enable automated detection of misuse." }, { "requirementID": "Practice 6.1: Monitor for evidence of misuse - 5", "requirementText": "Consider tiered methods of detection, such as scanning for misuse using less costly automated methods and then validating through direct human intervention, to help prioritize limited resources, improve privacy, and increase coverage." }, { "requirementID": "Practice 6.1: Monitor for evidence of misuse - 6", "requirementText": "Collaborate with other actors, such as content distribution platforms, downstream model adapters, cloud service providers, third-party researchers, and law enforcement officials, to track real-world incidents of misuse." } ] }, "AI RMF 1.0": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirements": [ { "requirementID": "MEASURE 2.4", "requirementText": "The functionality and behavior of the AI system and its components – as identified in the MAP function – are monitored when in production." }, { "requirementID": "MANAGE 3.1", "requirementText": "AI risks and benefits from third-party resources are regularly monitored, and risk controls are applied and documented." }, { "requirementID": "MANAGE 3.2", "requirementText": "Pre-trained models which are used for development are monitored as part of AI system regular monitoring and maintenance." }, { "requirementID": "MANAGE 4.1", "requirementText": "Post-deployment AI system monitoring plans are implemented, including mechanisms for capturing and evaluating input from users and other relevant AI actors, appeal and override, decommissioning, incident response, recovery, and change management." } ] }, "IR 8596: Cybersecurity Framework Profile for Artificial Intelligence (Cyber AI Profile): NIST Community Profile": { "link": "https://csrc.nist.gov/pubs/ir/8596/iprd", "requirements": [ { "requirementID": "PR.PS-04", "requirementText": "Log records are generated and made available for continuous monitoring" }, { "requirementID": "PR.PS-06", "requirementText": "Secure software development practices are integrated, and their performance is monitored throughout the software development life cycle" }, { "requirementID": "DE.CM-01", "requirementText": "Networks and network services are monitored to find potentially adverse events" }, { "requirementID": "DE.CM-02", "requirementText": "The physical environment is monitored to find potentially adverse events" }, { "requirementID": "DE.CM-03", "requirementText": "Personnel activity and technology usage are monitored to find potentially adverse events" }, { "requirementID": "DE.CM-06", "requirementText": "External service provider activities and services are monitored to find potentially adverse events" }, { "requirementID": "DE.CM-09", "requirementText": "Computing hardware and software, runtime environments, and their data are monitored to find potentially adverse events" }, { "requirementID": "DE.AE-02", "requirementText": "Potentially adverse events are analyzed to better understand associated activities" }, { "requirementID": "DE.AE-03", "requirementText": "Information is correlated from multiple sources" }, { "requirementID": "DE.AE-04", "requirementText": "The estimated impact and scope of adverse events are understood" }, { "requirementID": "RS.AN-03", "requirementText": "Analysis is performed to establish what has taken place during an incident and the root cause of the incident" }, { "requirementID": "RS.AN-07", "requirementText": "Incident data and metadata are collected, and their integrity and provenance are preserved" } ] }, "SP 800-218A": { "link": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirements": [ { "requirementID": "PO.5.1", "requirementText": "Separate and protect each environment involved in software development.\n\nMonitor, track, and limit resource usage and rates for AI model users during model development.\n\nOnly store sensitive data used during AI model development, including production data, within organization-approved environments and locations within those environments.\n\nProtect all training pipelines, model registries, and other components within the environments according to the principle of least privilege.\n\nContinuously monitor training-related activity in pipelines and model modifications in the model registry.\n\nFollow recommended practices for securely configuring each environment.\n\nContinuously monitor each environment for plaintext secrets." }, { "requirementID": "PO.5.3", "requirementText": "Continuously monitor software execution performance and behavior in software development environments to identify potential suspicious activity and other issues.\n\nPerform continuous security monitoring for all development environment components that host an AI model or related resources (e.g., model APIs, weights, configuration parameters, training datasets).\n\nContinuous monitoring and analysis tools should generate alerts when detected activity involving an AI model passes a risk threshold or otherwise merits additional investigation." }, { "requirementID": "RV.1.1", "requirementText": "Gather information from software acquirers, users, and public sources on potential vulnerabilities in the software and third-party components that the software uses, and investigate all credible reports.\n\nLog, monitor, and analyze all inputs and outputs for AI models to detect possible security and performance issues (see PO.5.3).\n\nMake the users of AI models aware of mechanisms for reporting potential security and performance issues.\n\nMonitor vulnerability and incident databases for information on AI-related concerns, including the machine learning frameworks and libraries used to build AI models." }, { "requirementID": "RV.2.1", "requirementText": "Analyze each vulnerability to gather sufficient information about risk to plan its remediation or other risk response." } ] } }, "OECD": { "Due Diligence Guidance for Responsible AI": { "link": "https://www.oecd.org/content/dam/oecd/en/publications/reports/2026/02/oecd-due-diligence-guidance-for-responsible-ai_7831bb49/41671712-en.pdf", "requirements": [ { "requirementID": "Step 4 - Track implementation and results of due diligence activities", "requirementText": "Track the implementation and effectiveness of the enterprise’s due diligence activities, i.e., its measures to identify, prevent, mitigate and, where appropriate, support remediation of adverse impacts." } ] } }, "OpenAI": { "Preparedness Framework": { "link": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "requirements": [ { "requirementID": "Safeguards Against Malicious Users - Usage Monitoring", "requirementText": "If a model does not refuse and provides assistance to harmful tasks, monitors can stop or catch malicious users before they have achieved an unacceptable scale of harm, through a combination of automated and human detection and enforcement within an acceptable time frame." } ] }, "Safety Best Practices": { "link": "https://platform.openai.com/docs/guides/safety-best-practices", "requirements": [ { "requirementID": "Implement safety identifiers", "requirementText": "Sending safety identifiers in your requests can be a useful tool to help OpenAI monitor and detect abuse. This allows OpenAI to provide your team with more actionable feedback in the event that we detect any policy violations in your application.\n\nA safety identifier should be a string that uniquely identifies each user. Hash the username or email address in order to avoid sending us any identifying information. If you offer a preview of your product to non-logged in users, you can send a session ID instead.\n\nInclude safety identifiers in your API requests with the safety_identifier parameter" } ] } }, "OWASP": { "LLM Top 10": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirements": [ { "requirementID": "LLM03: Supply Chain - 7", "requirementText": "Implement strict monitoring and auditing practices for collaborative model development environments to prevent and quickly detect any abuse. \"HuggingFace SF_Convertbot Scanner\" is an example of automated scripts to use." }, { "requirementID": "LLM04: Data and Model Poisoning - 9", "requirementText": "Monitor training loss and analyze model behavior for signs of poisoning. Use thresholds to detect anomalous outputs.\n" }, { "requirementID": "LLM05: Improper Data Handling - 7", "requirementText": "Implement robust logging and monitoring systems to detect unusual patterns in LLM outputs that might indicate exploitation attempts." }, { "requirementID": "LLM08: Vector and Embedding Weaknesses - 4", "requirementText": "Maintain detailed immutable logs of retrieval activities to detect and respond promptly to suspicious behavior.\n" }, { "requirementID": "LLM09: Misinformation - 4", "requirementText": "Implement tools and processes to automatically validate key outputs, especially output from high-stakes environments." }, { "requirementID": "LLM10: Unbounded Consumption - 7", "requirementText": "Continuously monitor resource usage and implement logging to detect and respond to unusual patterns of resource consumption." }, { "requirementID": "LLM10: Unbounded Consumption - 8", "requirementText": "Implement watermarking frameworks to embed and detect unauthorized use of LLM outputs." } ] }, "OWASP Model Context Protocol (MCP) Top 10": { "link": "https://owasp.org/www-project-mcp-top-10/", "requirements": [ { "requirementID": "MCP01:2025 - Token Mismanagement and Secret Exposure - 4", "requirementText": "Secure Context & Log Management\n- Redact or mask secrets before writing to logs or telemetry.\n- Store diagnostic traces in protected locations with strict access control.\n- Rotate and invalidate all tokens immediately upon suspected exposure." }, { "requirementID": "MCP03:2025 - Tool Poisoning - 10", "requirementText": "Conduct forensic analysis: which agents used the poisoned schema, what actions executed, which data changed or was removed." }, { "requirementID": "MCP07:2025 – Insufficient Authentication & Authorization - 6", "requirementText": "Logging, Monitoring & Auditing\n- Log every authentication attempt and authorization decision.\n- Detects repeated failed logins, invalid tokens, or cross-tenant token reuse.\n- Feed these logs into a SIEM/XDR for anomaly detection and alerting." }, { "requirementID": "MCP08:2025 – Lack of Audit and Telemetry - 1", "requirementText": "Implement Structured, Tamper-Evident Logging Log all agent actions, tool invocations, schema versions, and context snapshots in a structured format (JSON, CEF, OTEL). Apply cryptographic hashing (HMAC, SHA-256) to log files for integrity. Store logs in append-only or write-once media (e.g., AWS S3 Object Lock, WORM storage)." }, { "requirementID": "MCP08:2025 – Lack of Audit and Telemetry - 2", "requirementText": "Include essential fields:\ntimestamp\nagent_id\nsession_id\ntool_invoked\nparameters_used\nresponse_summary\nuser_identity (if applicable)" }, { "requirementID": "MCP08:2025 – Lack of Audit and Telemetry - 3", "requirementText": "Integrate with SIEM, XDR, or Centralized Monitoring\n- Forward MCP logs to enterprise SIEM systems (Splunk, ELK, Sentinel, Chronicle, etc.) for correlation.\n- Establish automated alert rules for high-risk activities (e.g., tool execution involving sensitive data).\n- Use Extended Detection and Response (XDR) systems to correlate agent behaviors with network or endpoint signals." }, { "requirementID": "MCP08:2025 – Lack of Audit and Telemetry - 4", "requirementText": "Protect Sensitive Data in Logs\n- Implement PII-safe logging: tokenize or mask user identifiers and redact sensitive fields before storage.\n- Use field-level encryption for secrets, tokens, or confidential context entries.\n- Apply data classification labels to log streams to govern retention and access." }, { "requirementID": "MCP08:2025 – Lack of Audit and Telemetry - 5", "requirementText": "Establish Behavioral Baselines\n- Collect telemetry to build a behavioral profile of normal agent operations.\n- Use anomaly detection or ML-based behavioral analytics to flag deviations (e.g., unexpected API calls, unusual output patterns).\n- Regularly review and update baseline thresholds." }, { "requirementID": "MCP08:2025 – Lack of Audit and Telemetry - 6", "requirementText": "Enforce Access Control & Segregation of Duties\n- Restrict who can access logs — separate operational monitoring from security investigations.\n- Require dual authorization for log deletion or retention changes.\n- Apply least privilege and auditing on logging subsystems themselves." }, { "requirementID": "MCP08:2025 – Lack of Audit and Telemetry - 7", "requirementText": "Implement Real-Time Observability\n- Use OpenTelemetry or equivalent frameworks to trace requests across the MCP pipeline — from prompt creation to tool invocation.\n- Tag every trace with session and schema identifiers to enable end-to-end correlation.\n- Display agent performance and behavior dashboards for operational visibility." }, { "requirementID": "MCP08:2025 – Lack of Audit and Telemetry - 8", "requirementText": "Retention & Compliance Policies\n- Align log retention with applicable frameworks (e.g., PCI DSS: 1 year minimum).\n- Automatically archive or purge logs per retention schedule.\n- Periodically verify that retention, encryption, and deletion processes function as intended." }, { "requirementID": "MCP08:2025 – Lack of Audit and Telemetry - 9", "requirementText": "Continuous Audit & Verification\n- Conduct periodic audit drills to ensure investigators can reconstruct events from logs.\n- Test integrity checks — attempt to tamper with logs and validate detection alerts.\n- Implement audit trail self-verification, where logs cross-reference session data for consistency." }, { "requirementID": "MCP08:2025 – Lack of Audit and Telemetry - 10", "requirementText": "Re-enable detailed logging at all MCP layers (agent, tool, and network). Deploy forwarders to send logs to central SIEM/XDR with retention guarantees. Implement masking and pseudonymization to balance privacy and audit needs. Reconstruct minimal timeline from external system logs (firewalls, proxies). Perform root-cause review and enforce mandatory logging for all MCP agents." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 5", "requirementText": "Monitor for Anomalous or Unauthorized Behavior\n- Correlate telemetry to identify new MCP-related API traffic or agent activity from unknown hosts.\n- Set up alerts for endpoints responding on MCP-standard routes (/mcp, /agent/tools, /context).\n- Track configuration drift and endpoint proliferation over time." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 8", "requirementText": "Detection and Response Integration\n- Include shadow MCP detection in threat-hunting playbooks.\n- Upon detection, trigger an incident response workflow to contain, image, and analyze the rogue server.\n- Track remediation metrics (mean time to discovery and closure)." }, { "requirementID": "MCP09:2025 – Shadow MCP Servers - 11", "requirementText": "Review logs and assess data exposure or leakage." } ] }, "OWASP Top 10 for Agentic Applications for 2026": { "link": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "requirements": [ { "requirementID": "ASI01: Agent Goal Hijack - 7", "requirementText": "Maintain comprehensive logging and continuous monitoring of agent activity, establishing a behavioral baseline that includes goal state, tool-use patterns, and invariant properties (e.g., schema, access patterns). Track a stable identifier for the active goal where feasible, and alert on Page 11genai.owasp.org any deviations-such as unexpected goal changes, anomalous tool sequences, or shifts from the established baseline-so that unauthorized goal drift is immediately visible in operations." }, { "requirementID": "ASI02: Tool Misuse and Exploitation - 8", "requirementText": "Logging, Monitoring, and Drift Detection. Maintain immutable logs of all tool invocations and parameter changes. Continuously monitor for anomalous execution rates, unusual tool-chaining patterns (e.g., DB read followed by external transfer), and policy violations." }, { "requirementID": "ASI03: Identity and Privilege Abuse - 9", "requirementText": "Detect abnormal cross-agent privilege elevation and device-code style phishing flows by monitoring when agents request new scopes or reuse tokens outside their original, signed intent." }, { "requirementID": "ASI04: Agentic Supply Chain Vulnerabilities - 4", "requirementText": "Secure prompts and memory: Put prompts, orchestration scripts, and memory schemas under version control with peer review; scan for anomalies." }, { "requirementID": "ASI04: Agentic Supply Chain Vulnerabilities - 6", "requirementText": "Continuous validation and monitoring: Re-check signatures, hashes, and SBOMs (incl. AIBOMs) at runtime; monitor behavior, privilege use, lineage, and inter-module telemetry for anomalies." }, { "requirementID": "ASI05: Unexpected Code Execution (RCE) - 7", "requirementText": "Code analysis and monitoring: Do static scans before execution; enable runtime monitoring; watch for prompt-injection patterns; log and audit all generation and runs." }, { "requirementID": "ASI06: Memory & Context Poisoning - 2", "requirementText": "Content validation: Scan all new memory writes and model outputs (rules + AI) for malicious or sensitive content before commit." }, { "requirementID": "ASI08: Cascading Failures - 8", "requirementText": "Behavioral and governance drift detection: Track decisions vs baselines and alignment; flag gradual degradation." }, { "requirementID": "ASI08: Cascading Failures - 10", "requirementText": "Logging and non-repudiation. Record all inter-agent messages, policy decisions, and execution outcomes in tamper-evident, time-stamped logs bound to cryptographic agent identities. Maintain lineage metadata for every propagated action to support forensic traceability, rollback validation, and accountability during cascades." }, { "requirementID": "ASI09: Human-Agent Trust Exploitation - 2", "requirementText": "Immutable logs: Keep tamper-proof records of user queries and agent actions for audit and forensics." }, { "requirementID": "ASI09: Human-Agent Trust Exploitation - 3", "requirementText": "Behavioral detection: Monitor sensitive data being exposed in either conversations or Agentic connections, as well as risky action executions over time." }, { "requirementID": "ASI09: Human-Agent Trust Exploitation - 9", "requirementText": "Plan-divergence detection: Compare agent action sequences against approved workflow baselines and alert when unusual detours, skipped validation steps, or novel tool combinations indicate possible deception or drift." }, { "requirementID": "ASI10: Rogue Agents - 1", "requirementText": "Governance & Logging: Maintain comprehensive, immutable and signed audit logs of all agent actions, tool calls, and inter-agent communication to review for stealth infiltration or unapproved delegation." }, { "requirementID": "ASI10: Rogue Agents - 3", "requirementText": "Monitoring & Detection: Deploy behavioral detection, such as watchdog agents to validate peer behavior and outputs, focusing on detecting collusion patterns and coordinated false signals. Monitor for anomalies such as excessive or abnormal actions executions." } ] } }, "Personal Data Protection Commission Singapore (PDPC)": { "Model Artificial Intelligence Governance Framework Second Edition": { "link": "https://www.pdpc.gov.sg/-/media/files/pdpc/pdf-files/resource-for-organisation/ai/sgmodelaigovframework2.pdf", "requirements": [ { "requirementID": "1. Clear roles and responsibilities for the ethical deployment of AI - c) (ii)", "requirementText": "Maintenance, monitoring, documentation and review of the AI models that have been deployed, with a view to taking remediation measures where needed." }, { "requirementID": "2. Risk management and internal controls - b) (ii)", "requirementText": "Establishing monitoring and reporting systems as well as processes to ensure that the appropriate level of management is aware of the performance of and other issues relating to the deployed AI. Where appropriate, the monitoring can include autonomous monitoring to effectively scale human oversight. AI systems can be designed to report on the confidence level of their predictions, and explainability features could focus on why the AI model had a certain level of confidence." }, { "requirementID": "Repeatability - b)", "requirementText": "Assessing how exceptions can be identified and handled when decisions are not repeatable, e.g. when randomness has been introduced by design;" }, { "requirementID": "Repeatability - e)", "requirementText": "Identifying and accounting for changes over time to ensure that models trained on time-sensitive data remain relevant." } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirements": [ { "requirementID": "7.9", "requirementText": "An Entity may develop monitoring controls to measure the fairness of an Entity's Al Model and policies when and how to initiate remedial actions." }, { "requirementID": "12.7", "requirementText": "An Entity must put in place proper reporting and monitoring mechanisms to ensure that the integrity and quality of work conducted by the outsourcing service provider is maintained." }, { "requirementID": "13.6.4", "requirementText": "Entities must set in-built limits and link to warning levels or auto-close routines." }, { "requirementID": "13.7.2", "requirementText": "Human monitoring must provide information that a Supervisor can respond to in a manageable time frame to adjust parameters during the operation of the Algorithm." }, { "requirementID": "15.15", "requirementText": "To the extent that it is strictly necessary for the purposes of ensuring Bias monitoring, detection, and correction in relation to the High-Risk Al Systems, the Providers of such systems may process personal data, subject to compliance with the Qatar Law No. (13) of 2016 on Personal Data Privacy Protection. To the extent feasible, using of security and privacy-preserving measures, such as pseudonymization, or encryption where anonymization is used." }, { "requirementID": "17.2", "requirementText": "An Entity must develop a program for Al Trust, Risk and Security Management (AI TRiSM). The program must include:\n- Toolsets for content anomaly detection.\n- Toolsets for data protection from Providers and other third-parties.\n- Toolsets for third-party system security.\n- Policy for the acceptable use of Al.\n- Processes for assessment of privacy, fairness and bias." }, { "requirementID": "17.9", "requirementText": "An Entity that utilizes Al must deploy tools for content anomaly detection." }, { "requirementID": "19.1", "requirementText": "An Entity must support the Monitoring Systems and plans for High-Risk Al Systems. This means for its own Al Systems but also accessing the information provided of all activities carried out by Providers of Al Systems to collect and review experience gained from the use of Al Systems." }, { "requirementID": "19.2", "requirementText": "The Entity must ensure the compliance of the Provider to its obligations to evaluate the conformance to the predicted model outcomes of Al Systems throughout their Life Cycle, the Monitoring System must collect, document, and analyze relevant data, which may be provided by Users, or which may be collected through other sources on the performance of High-Risk Al Systems." }, { "requirementID": "19.3", "requirementText": "The Entity should maintain records of its experience with Al Systems which are auditable and where relevant and considering the type of system used, should maintain on-going and up-to-date information." }, { "requirementID": "19.3.1", "requirementText": "Establish audit logs and maintaining traceability of decisions and outcomes of the Al System." } ] } }, "SANS": { "Critical AI Security Guidelines": { "link": "https://sansorg.egnyte.com/dl/bvkYQxrW8QMj", "requirements": [ { "requirementID": "5 Monitoring", "requirementText": "Effective monitoring is essential to maintaining AI security over time. AI models and systems must be continuously observed for performance degradation, adversarial attacks, and unauthorized access. Implementing logging, anomaly detection, and drift monitoring ensures AI applications remain reliable and aligned with intended behaviors." }, { "requirementID": "7.1 Capture Audit Trails Across the Stack", "requirementText": "Ensure logs include prompt inputs, augmentation sources (like vectorDB queries), model outputs, function calls, and tool invocations." }, { "requirementID": "7.2 Monitor for Indicators of Model Tampering", "requirementText": "Watch for sudden changes in inference behavior, drift in outputs, or increased refusal rates, which may indicate adversarial manipulation or unauthorized updates." }, { "requirementID": "7.3 Employ Detection on Prompt and Output Layers", "requirementText": "Include pattern-based and behavioral monitoring to identify jailbreak attempts, abuse of multilingual prompts, or bypasses via encoding/compression." } ] } }, "SDAIA (Saudi Arabia)": { "AI Ethics Principles": { "link": "https://sdaia.gov.sa/en/SDAIA/about/Documents/ai-principles.pdf", "requirements": [ { "requirementID": "Principle 2 – Privacy & Security - Deploy and Monitor - 1", "requirementText": "After the deployment of the AI system, when its outcomes are realized, there must be continuous monitoring to ensure that the AI system is privacy-preserving, safe and secure. The privacy impact assessment and risk management assessment should be continuously revisited to ensure that societal and ethical considerations are regularly evaluated." }, { "requirementID": "Principle 5 – Reliability & Safety - Deploy and Monitor - 1", "requirementText": "Monitoring the robustness of the AI system should be adopted and undertaken in a periodic and continuous manner to measure and assess any risks related to the technicalities of the AI system (an inward perspective) as well as the magnitude of the risk posed by the system and its capabilities (an outward perspective)." }, { "requirementID": "Principle 5 – Reliability & Safety - Deploy and Monitor - 2", "requirementText": "The model must also be monitored in a periodic and continuous manner to verify whether its operations and functions are compatible with the designed structure and frameworks. The AI system must also be safe to prevent destructive use to exploit its data and results to harm entities, individuals, or groups. It is necessary to continuously work on implementation and development to ensure system reliability." }, { "requirementID": "Principle 7 – Accountability & Responsibility - Deploy and Monitor - 1", "requirementText": "The responsibility and associated liability in the Deploy and Monitor step should be set clearly. The outcomes and decisions set in the build and validate step should be monitored continuously and should result in periodic performance reports." }, { "requirementID": "Principle 7 – Accountability & Responsibility - Deploy and Monitor - 2", "requirementText": "Predefined triggers/alerts should be defined for this step on the data and performance metrics. Setting these triggers is a rigorous process and each trigger should be assigned to the appropriate stakeholder. These triggers/alerts can be defined as part of the risk mitigation or disaster recovery procedure and may need human oversight." } ] } }, "Smart Dubai (UAE)": { "AI Ethics Principles & Guidelines": { "link": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirements": [ { "requirementID": "1.2.2.7", "requirementText": "AI operator organisations should consider working with their vendors (AI developer organisations) to continually monitor performance." }, { "requirementID": "1.3.1.3", "requirementText": "Where possible given the model design, AI developer organisations should consider building in a means by which the “decision journey” of a specific outcome (i.e. the component decisions leading to it) can be logged." } ] } }, "World Economic Forum": { "Presidio AI Framework: Towards Safe Generative AI Models": { "link": "https://www3.weforum.org/docs/WEF_Presidio_AI%20Framework_2024.pdf", "requirements": [ { "requirementID": "Model drift monitoring and watermarking", "requirementText": "A critical goal of the adaptation phase is to ensure that the adapted model remains effective and aligned with the selected use case. Model drift monitoring involves regularly comparing postdeployment metrics to maintain performance in the face of evolving data, adversarial inputs, noise and external factors. The goal is to mitigate the risk of model drift, where the model’s output deviates from expectations over time. Best practices include systematically using data, algorithms, and tools for tracking data drift, and defining response protocols and adaptation techniques to sustain model performance and customer trust." } ] } } }, "Principle 13": { "CEN/CENELEC": { "prEN 40000-1-2: Cybersecurity requirements for products with digital elements - Part 1-2: Principles for cyber resilience": { "link": "https://genorma.com/en/standards/pren-40000-1-2", "requirements": [ { "requirementID": "7.10.", "requirementText": "Planning for secure decommissioning" } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirements": [ { "requirementID": "DCS-01", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for the secure disposal of equipment used outside the organization's premises. If the equipment is not physically destroyed a data destruction procedure that renders recovery of information impossible must be applied. Review and update the policies and procedures at least annually, or upon significant changes." }, { "requirementID": "DCS-02", "requirementText": "Establish, document, approve, communicate, apply, evaluate and maintain policies and procedures for the relocation or transfer of hardware, software, or data/information to an offsite or alternate location. The relocation or transfer request requires the written or cryptographically verifiable authorization. Review and update the policies and procedures at least annually, or upon significant changes." }, { "requirementID": "DSP-02", "requirementText": "Apply industry accepted methods for the secure disposal of data from storage media such that data is not recoverable by any forensic means." } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirements": [ { "requirementID": "3.2.1 Asset Management for AI/ML Systems - 5", "requirementText": "The entity should have a process to decommission and securely dispose of AI/ML assets when they reach their end of life, ensuring that appropriate data is securely wiped and cannot be recovered." } ] } }, "ETSI": { "EN 304 223 - Securing Artificial Intelligence (SAI); Baseline Cyber Security Requirements for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_en/304200_304299/304223/02.01.01_60/en_304223v020101p.pdf", "requirements": [ { "requirementID": "Provision 5.5.1-1", "requirementText": "If a Developer or System Operator decides to transfer or share ownership of training data and/or a model to another entity they shall involve Data Custodians and securely dispose of these assets. This will protect AI against security issues that can transfer from one AI system instantiation to another." }, { "requirementID": "Provision 5.5.1-2", "requirementText": "If a Developer or System Operators decides to decommission a model and/or system, they shall involve Data Custodians and securely delete applicable data and configuration details." } ] }, "TR 104 128 - Securing Artificial Intelligence (SAI); Guide to Cyber Security for AI Models and Systems": { "link": "https://www.etsi.org/deliver/etsi_tr/104100_104199/104128/01.01.01_60/tr_104128v010101p.pdf", "requirements": [ { "requirementID": "Provision 5.5.1-1", "requirementText": "\"If a Developer or System Operator decides to transfer or share ownership of training data and/or a model to another entity they shall involve Data Custodians and securely dispose of these assets. This will protect AI against security issues that can transfer from one AI system instantiation to another.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nImproper disposal or transfer can lead to unauthorized data recovery, risking breaches, IP loss, and non-compliance with data protection laws.\n\nExample Measures/Controls:\nDevelop and Implement a Secure Transfer and Disposal Policy with Data Custodian Oversight: Establish a comprehensive policy to govern the secure transfer and disposal of training data and models. Ensure that Data Custodians oversee all actions, confirming compliance with regulatory standards, protection of intellectual property, and adherence to organizational policies. This includes compliance with GDPR when involving personal data." }, { "requirementID": "Provision 5.5.1-2", "requirementText": "\"If a Developer or System Operators decides to decommission a model and/or system, they shall involve Data Custodians and securely delete applicable data and configuration details.\" (ETSI TS 104 223 [i.1])\n\nRelated threats/risks:\nInsecure data deletion during decommissioning can lead to unauthorized access to residual data, increasing regulatory and security risks.\n\nExample Measures/Controls:\nImplement a Secure Data Deletion Policy with Data Custodian Oversight: Establish a policy for securely deleting data and models during decommissioning, specifying methods compliant with standards. Ensure Data Custodians validate all deletions to maintain regulatory compliance and traceability." } ] } }, "NCSC/NSA/CISA etc": { "AI Data Security\n": { "link": "https://media.defense.gov/2025/May/22/2003720601/-1/-1/0/CSI_AI_DATA_SECURITY.PDF", "requirements": [ { "requirementID": "1.9 Delete Data Securely", "requirementText": "Prior to repurposing or decommissioning any functional drives used for AI data storage and processing, erase them using a secure deletion method such as cryptographic erase, block erase, or data overwrite. Refer to NIST SP 800-88, “Guidelines for Media Sanitization,” [20] for guidance on appropriate deletion methods." } ] } }, "NIST": { "AI RMF 1.0": { "link": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirements": [ { "requirementID": "GOVERN 1.7", "requirementText": "Processes and procedures are in place for decommissioning and phasing out AI systems safely and in a manner that does not increase risks or decrease the organization’s trustworthiness." } ] } } }, "Outlier": { "Central Bank of the UAE": { "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E": { "link": "Guidance Note on the Consumer Protection and Responsible Adoption and Use of Artificial Intelligence and Machine Learning by Licensed Financial Institutions in the U.A.E", "requirements": [ { "requirementID": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirementText": "2. Governance and Accountability - a" }, { "requirementID": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirementText": "3. Fairness/Non-Discrimination and Ethics - a" }, { "requirementID": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirementText": "3. Fairness/Non-Discrimination and Ethics - b" }, { "requirementID": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirementText": "4. Transparency and Explain ability - c" }, { "requirementID": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirementText": "4. Transparency and Explain ability - e" }, { "requirementID": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirementText": "5. Data Quality, Privacy and Security - e" }, { "requirementID": "https://rulebook.centralbank.ae/en/rulebook/guidance-note-consumer-protection-and-responsible-adoption-and-use-artificial-intelligence", "requirementText": "9. Outsourcing and Third-Party Risk - e" } ] } }, "CISA": { "Principles for the Secure Integration of Artificial Intelligence in Operational Technology": { "link": "Principles for the Secure Integration of Artificial Intelligence in Operational Technology", "requirements": [ { "requirementID": "https://www.cisa.gov/sites/default/files/2026-01/joint-guidance-principles-for-the-secure-integration-of-artificial-intelligence-in-operational-technology-508cV2.pdf", "requirementText": "2.2.2 - Data Sovereignty" } ] } }, "Cloud Security Alliance (CSA)": { "AI Controls Matrix": { "link": "AI Controls Matrix", "requirements": [ { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "A&A-04" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "AIS-09" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "BCR-06" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "CCC-05" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "DCS-03" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "DCS-08" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "DCS-12" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "DCS-13" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "DCS-15" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "DSP-13" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "DSP-14" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "GRC-04" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "HRS-01" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "HRS-03" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "HRS-06" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "HRS-07" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "HRS-08" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "IAM-02" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "IAM-15" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "IPY-02" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "LOG-06" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "SEF-08" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "STA-05" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "STA-10" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "STA-12" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "UEM-03" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "UEM-06" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "UEM-07" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "UEM-11" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "UEM-12" }, { "requirementID": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix", "requirementText": "UEM-13" } ] } }, "CoSAI": { "AI Incident Response Framework": { "link": "AI Incident Response Framework", "requirements": [ { "requirementID": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirementText": "3.3.2. Detection and Analysis Phase - Initial Triage - Impact Assessment" }, { "requirementID": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirementText": "3.3.2. Detection and Analysis Phase - Investigation Procedures - Attribution Assessment" }, { "requirementID": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirementText": "3.3.3. Containment, Eradication, and Recovery Phase - Recovery Procedures - Service Restoration" }, { "requirementID": "https://github.com/cosai-oasis/ws2-defenders/blob/main/incident-response/AI%20Incident%20Response.md", "requirementText": "3.3.3. Containment, Eradication, and Recovery Phase - Recovery Procedures - Business Continuity" } ] } }, "Cyber Security Council (UAE)": { "National Cyber Security Policy for Artificial Intelligence": { "link": "National Cyber Security Policy for Artificial Intelligence", "requirements": [ { "requirementID": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirementText": "3.1.1 Cyber Security Policies & Procedures - 3" }, { "requirementID": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirementText": "3.6.3 Digital Forensics for AI/ML Security Incidents - 2" }, { "requirementID": "https://csc.gov.ae/documents/38662/0/National+Cyber+Security+Policy+for+Artificial+Intelligence_v1.1.pdf/4e02b32e-9f62-948d-4bc8-b580d596451b?t=1766994254544", "requirementText": "3.6.3 Digital Forensics for AI/ML Security Incidents - 4" } ] } }, "Databricks": { "The Databricks AI Security Framework": { "link": "The Databricks AI Security Framework", "requirements": [ { "requirementID": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirementText": "DASF 13: Use near real-time data\n" }, { "requirementID": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirementText": "DASF 15: Explore datasets and identify problems\n" }, { "requirementID": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirementText": "DASF 26: Fine-tune large language models (LLMs)\n" }, { "requirementID": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirementText": "DASF 27: Pretrain a large language model (LLM)" }, { "requirementID": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirementText": "DASF 42: Employ data-centric MLOps and LLMOps\n" }, { "requirementID": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirementText": "DASF 44: Triggering actions in response to a specific event\n" }, { "requirementID": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirementText": "DASF 48: Use hardened Runtime for Machine Learning\n" }, { "requirementID": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirementText": "DASF 69: Securely host Custom MCP Servers" }, { "requirementID": "https://www.databricks.com/sites/default/files/2025-02/databricks-ebook-dasf-2.pdf", "requirementText": "DASF 70: Securely connect to External MCP Servers" } ] } }, "ENISA": { "Multilayer Framework for Good Cybersecurity Practices for AI": { "link": "Multilayer Framework for Good Cybersecurity Practices for AI", "requirements": [ { "requirementID": "https://www.enisa.europa.eu/sites/default/files/publications/Multilayer%20Framework%20for%20Good%20Cybersecurity%20Practices%20for%20AI.pdf", "requirementText": "Networking 9" }, { "requirementID": "https://www.enisa.europa.eu/sites/default/files/publications/Multilayer%20Framework%20for%20Good%20Cybersecurity%20Practices%20for%20AI.pdf", "requirementText": "Regulation 5" }, { "requirementID": "https://www.enisa.europa.eu/sites/default/files/publications/Multilayer%20Framework%20for%20Good%20Cybersecurity%20Practices%20for%20AI.pdf", "requirementText": "Regulation 6" } ] } }, "EU ": { "EU AI Act": { "link": "EU AI Act", "requirements": [ { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "10.3 Data and Data Governance" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "10.4 Data and Data Governance" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "10.5 Data and Data Governance" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "10.6 Data and Data Governance" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "14.5 Human Oversight" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "17.3 Quality Management Systems" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "26.5 Obligations of deployers of high-risk AI systems" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "26.10 Obligations of deployers of high-risk AI systems" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "26.12 Obligations of deployers of high-risk AI systems" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "50.3 Transparency obligations for providers and deployers of certain AI systems" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "50.6 Transparency obligations for providers and deployers of certain AI systems" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "50.7 Transparency obligations for providers and deployers of certain AI systems" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "53.3 Obligations for providers of general purpose AI models" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "53.4 Obligations for providers of general purpose AI models" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "53.6 Obligations for providers of general purpose AI models" }, { "requirementID": "https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=OJ:L_202401689", "requirementText": "53.7 Obligations for providers of general purpose AI models" } ] } }, "European Commission": { "Assessment List for Trustworthy Artificial Intelligence (ALTAI)": { "link": "Assessment List for Trustworthy Artificial Intelligence (ALTAI)", "requirements": [ { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/assessment-list-trustworthy-artificial-intelligence-altai-self-assessment", "requirementText": "REQUIREMENT #5 Diversity, Non-discrimination\nand Fairness" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/assessment-list-trustworthy-artificial-intelligence-altai-self-assessment", "requirementText": "REQUIREMENT #6 Societal and Environmental\nWell-being" } ] }, "Ethics guidelines for trustworthy AI": { "link": "Ethics guidelines for trustworthy AI", "requirements": [ { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "1.1.1 Fundamental Rights" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "1.5.2 Accessibility and universal design" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "1.5.3 Stakeholder Participation" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "1.6.1 Sustainable and Environmentally Friendly AI" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "1.6.2 Social Impact" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "1.6.3 Society and Democracy" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "1.7.3 Trade-offs" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "1.7.4 Redress" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "2.1.1 Architectures for Trustworthy AI" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "2.2.2 Codes of Conudct" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "2.2.5 Accountability via Governance Frameworks" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "2.2.7 Stakeholder Participation and Social Dialogue" }, { "requirementID": "https://digital-strategy.ec.europa.eu/en/library/ethics-guidelines-trustworthy-ai", "requirementText": "2.2.8 Diversity and Inclusive Design Teams" } ] } }, "Google": { "Secure AI Framework": { "link": "Secure AI Framework", "requirements": [ { "requirementID": "https://www.saif.google/secure-ai-framework", "requirementText": "Output Validation and Sanitization" } ] } }, "ICO": { "Guidance on the AI Auditing Framework - Draft guidance for consultation ": { "link": "Guidance on the AI Auditing Framework - Draft guidance for consultation ", "requirements": [ { "requirementID": "https://ico.org.uk/media2/about-the-ico/consultations/2617219/guidance-on-the-ai-auditing-framework-draft-for-consultation.pdf", "requirementText": "Statistical accuracy and discrimination" } ] } }, "ISO/IEC": { "DIS 27090": { "link": "DIS 27090", "requirements": [ { "requirementID": "https://www.iso.org/obp/ui/en/#iso:std:iso-iec:27090:dis:ed-1:v1:en", "requirementText": "7.7" } ] } }, "METI (Japan)": { "Governance Guidelines for Implementation of AI Principles": { "link": "Governance Guidelines for Implementation of AI Principles", "requirements": [ { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20220128_2.pdf", "requirementText": "Action Target 2-1" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20220128_2.pdf", "requirementText": "Action Target 3-3-1" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20220128_2.pdf", "requirementText": "Action Target 3-4-1" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20220128_2.pdf", "requirementText": "Action Target 4-1" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20220128_2.pdf", "requirementText": "Action Target 5-2" } ] } }, "MIC/METI (Japan)": { "AI Guidelines for Business": { "link": "AI Guidelines for Business", "requirements": [ { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Human-Centric - 1 (a)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Human-Centric - 1 (b)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Human-Centric - 1 (c)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Human-Centric - 2 (a)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Human-Centric - 2 (c)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Human-Centric - 6" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Fairness - 1 (b)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Fairness - 2 (b)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Fairness - 2 (c)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Privacy protection - 1 (a)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Privacy protection - 1 (b)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Transparency - 3 (b)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Transparency - 3 (c)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Education/literacy - 2 (a)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Innovation - 1 (a)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Innovation - 1 (b)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Innovation - 2 (a)" }, { "requirementID": "https://www.meti.go.jp/shingikai/mono_info_service/ai_shakai_jisso/pdf/20240419_9.pdf", "requirementText": "Innovation - 2 (b)" } ] } }, "Microsoft": { "Responsible AI Standard": { "link": "Responsible AI Standard", "requirements": [ { "requirementID": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirementText": "A1.1" }, { "requirementID": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirementText": "A1.2" }, { "requirementID": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirementText": "A1.3" }, { "requirementID": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirementText": "F1.1" }, { "requirementID": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirementText": "F2.1" }, { "requirementID": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirementText": "F2.9" }, { "requirementID": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirementText": "F3.1" }, { "requirementID": "https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/microsoft/final/en-us/microsoft-brand/documents/Microsoft-Responsible-AI-Standard-General-Requirements.pdf?culture=en-us&country=us", "requirementText": "RS3.9" } ] } }, "MITRE": { "ATLAS Framework": { "link": "ATLAS Framework", "requirements": [ { "requirementID": "https://atlas.mitre.org/mitigations", "requirementText": "AML.M0000 - Limit Public Release of Information" }, { "requirementID": "https://atlas.mitre.org/mitigations", "requirementText": "AML.M0001 - Limit Model Artifact Release" }, { "requirementID": "https://atlas.mitre.org/mitigations", "requirementText": "AML.M0034 - Deepfake Detection" } ] }, "SAFE-AI": { "link": "SAFE-AI", "requirements": [ { "requirementID": "https://atlas.mitre.org/pdf-files/SAFEAI_Full_Report.pdf", "requirementText": "Configuration errors" } ] } }, "Multi Agency": { "Guidelines for secure AI system development": { "link": "Guidelines for secure AI system development", "requirements": [ { "requirementID": "https://www.ncsc.gov.uk/files/Guidelines-for-secure-AI-system-development.pdf", "requirementText": "Manage your technical debt" } ] } }, "NCSC/NSA/CISA etc": { "AI Data Security\n": { "link": "AI Data Security\n", "requirements": [ { "requirementID": "https://media.defense.gov/2025/May/22/2003720601/-1/-1/0/CSI_AI_DATA_SECURITY.PDF", "requirementText": "1.8 Leverage privacy-preserving techniques" } ] } }, "NIST": { "AI 100-2e2025: Adversarial Machine Learning\nA Taxonomy and Terminology of Attacks and Mitigations": { "link": "AI 100-2e2025: Adversarial Machine Learning\nA Taxonomy and Terminology of Attacks and Mitigations", "requirements": [ { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-2e2025.pdf", "requirementText": "3.3.3 Direct Prompting Attacks - Interventions during deployment (5) - Prompt stealing detection" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-2e2025.pdf", "requirementText": "3.3.3 Direct Prompting Attacks - Interventions during deployment (5) - Input modification" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-2e2025.pdf", "requirementText": "3.3.3 Direct Prompting Attacks - Interventions during deployment (5) - Aggregating outputfrommultiple prompts" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-2e2025.pdf", "requirementText": "3.3.3 Direct Prompting Attacks - Unlearning" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-2e2025.pdf", "requirementText": "3.3.3 Direct Prompting Attacks - Watermarking" } ] }, "AI 800-1": { "link": "AI 800-1", "requirements": [ { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirementText": "Practice 1.2 Create threat profiles - 1" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirementText": "Practice 1.2 Create threat profiles - 2" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirementText": "Practice 3.1: Assess misuse risk from threat actors gaining unauthorized access to the model - 2" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirementText": "Practice 3.2: Maintain security practices sufficient to prevent unauthorized access - 1" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirementText": "Practice 4.1: Evaluate model capabilities on tasks relevant to assessing misuse risk - 7" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirementText": "Practice 4.1: Evaluate model capabilities on tasks relevant to assessing misuse risk - 8" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirementText": "Practice 4.2: Red-team safeguards - 6" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirementText": "Practice 4.2: Red-team safeguards - 9" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirementText": "Practice 6.4: Provide safe harbors for third-party safety research - 3" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirementText": "Practice 7.1: Publish transparency reports - 7" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirementText": "Practice 7.3: Report misuse incidents - 1" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.800-1.ipd2.pdf", "requirementText": "Practice 7.3: Report misuse incidents - 2" } ] }, "AI RMF 1.0": { "link": "AI RMF 1.0", "requirements": [ { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirementText": "MEASURE 2.11" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "requirementText": "MEASURE 2.12" } ] }, "SP 800-218A": { "link": "SP 800-218A", "requirements": [ { "requirementID": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirementText": "PW.4.2" }, { "requirementID": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-218A.pdf", "requirementText": "PW.6.1" } ] } }, "OECD": { "Due Diligence Guidance for Responsible AI": { "link": "Due Diligence Guidance for Responsible AI", "requirements": [ { "requirementID": "https://www.oecd.org/content/dam/oecd/en/publications/reports/2026/02/oecd-due-diligence-guidance-for-responsible-ai_7831bb49/41671712-en.pdf", "requirementText": "Step 1.2 - Internal management systems" }, { "requirementID": "https://www.oecd.org/content/dam/oecd/en/publications/reports/2026/02/oecd-due-diligence-guidance-for-responsible-ai_7831bb49/41671712-en.pdf", "requirementText": "Step 1.3 - Expectations on business relationships" } ] } }, "OpenAI": { "Preparedness Framework": { "link": "Preparedness Framework", "requirements": [ { "requirementID": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "requirementText": "Long-range Autonomy" }, { "requirementID": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "requirementText": "Sandbagging" }, { "requirementID": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "requirementText": "Safeguards Against a Misaligned Model - Value Alignment" }, { "requirementID": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "requirementText": "Safeguards Against a Misaligned Model - Instruction Alignment" } ] }, "Safety Best Practices": { "link": "Safety Best Practices", "requirements": [ { "requirementID": "https://platform.openai.com/docs/guides/safety-best-practices", "requirementText": "“Know your customer” (KYC)" } ] } }, "OWASP": { "AI Exchange": { "link": "AI Exchange", "requirements": [ { "requirementID": "https://owaspai.org/docs/ai_security_overview/", "requirementText": "1.1 General governance controls - AI PROGRAM" }, { "requirementID": "https://owaspai.org/docs/ai_security_overview/", "requirementText": "1.2 General controls for sensitive data limitation - DISCRETE" } ] }, "LLM Top 10": { "link": "LLM Top 10", "requirements": [ { "requirementID": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirementText": "LLM02: Sensitive Information Disclosure - 10" }, { "requirementID": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirementText": "LLM04: Data and Model Poisoning - 7" }, { "requirementID": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirementText": "LLM04: Data and Model Poisoning - 10" }, { "requirementID": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirementText": "LLM05: Improper Data Handling - 3" }, { "requirementID": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirementText": "LLM05: Improper Data Handling - 4" }, { "requirementID": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirementText": "LLM09: Misinformation - 1" }, { "requirementID": "https://genai.owasp.org/resource/owasp-top-10-for-llm-applications-2025/", "requirementText": "LLM09: Misinformation - 2" } ] }, "OWASP Model Context Protocol (MCP) Top 10": { "link": "OWASP Model Context Protocol (MCP) Top 10", "requirements": [ { "requirementID": "https://owasp.org/www-project-mcp-top-10/", "requirementText": "MCP02:2025 - Privilege Escalation via Scope Creep - 2" }, { "requirementID": "https://owasp.org/www-project-mcp-top-10/", "requirementText": "MCP02:2025 - Privilege Escalation via Scope Creep - 4" }, { "requirementID": "https://owasp.org/www-project-mcp-top-10/", "requirementText": "MCP03:2025 - Tool Poisoning - 4" }, { "requirementID": "https://owasp.org/www-project-mcp-top-10/", "requirementText": "MCP03:2025 - Tool Poisoning - 7" }, { "requirementID": "https://owasp.org/www-project-mcp-top-10/", "requirementText": "MCP06:2025 – Intent Flow Subversion - 1" } ] } }, "Personal Data Protection Commission Singapore (PDPC)": { "Model Artificial Intelligence Governance Framework Second Edition": { "link": "Model Artificial Intelligence Governance Framework Second Edition", "requirements": [ { "requirementID": "https://www.pdpc.gov.sg/-/media/files/pdpc/pdf-files/resource-for-organisation/ai/sgmodelaigovframework2.pdf", "requirementText": "Data for Model Development - c) Minimising inherent bias" }, { "requirementID": "https://www.pdpc.gov.sg/-/media/files/pdpc/pdf-files/resource-for-organisation/ai/sgmodelaigovframework2.pdf", "requirementText": "Repeatability - d)" }, { "requirementID": "https://www.pdpc.gov.sg/-/media/files/pdpc/pdf-files/resource-for-organisation/ai/sgmodelaigovframework2.pdf", "requirementText": "Reproducibility - e)" } ] } }, "Qatar Central Bank": { "Artificial Intelligence Guidelines": { "link": "Artificial Intelligence Guidelines", "requirements": [ { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "7.7" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "7.8" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "7.10" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "7.11" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "7.12" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "8.2.4" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "10.2" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "10.3" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "10.6" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "11.1" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "11.2" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "11.3" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "11.4" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "12.2" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "12.4" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "12.10" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "12.11" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "13.5.1" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "13.6.1" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "14.1" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "14.2" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "14.2.1" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "14.3" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "15.7" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "15.14" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "17.1" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "20.7" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "20.10" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "20.11" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "21.2" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "22.1" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "22.2" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "23.1" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "23.2" }, { "requirementID": "https://www.qcb.gov.qa/Services/Financial%20Technology/QCB_Artificial_Intelligence_Guideline.pdf", "requirementText": "24.1" } ] } }, "SANS": { "Critical AI Security Guidelines": { "link": "Critical AI Security Guidelines", "requirements": [ { "requirementID": "https://sansorg.egnyte.com/dl/bvkYQxrW8QMj", "requirementText": "3.5 Do Not Share Critical Models" }, { "requirementID": "https://sansorg.egnyte.com/dl/bvkYQxrW8QMj", "requirementText": "6.2 The Biggest Risk of AI Is Not Using AI" }, { "requirementID": "https://sansorg.egnyte.com/dl/bvkYQxrW8QMj", "requirementText": "6.5 Account for AI Security and Regulatory Frameworks" } ] } }, "SDAIA (Saudi Arabia)": { "AI Adoption Framework": { "link": "AI Adoption Framework", "requirements": [ { "requirementID": "https://sdaia.gov.sa/en/SDAIA/about/Files/AIAdoptionFramework.pdf", "requirementText": "5.1.2 Privacy and Safety - Complying with National Frameworks" } ] }, "AI Ethics Principles": { "link": "AI Ethics Principles", "requirements": [ { "requirementID": "https://sdaia.gov.sa/en/SDAIA/about/Documents/ai-principles.pdf", "requirementText": "Principle 6 – Transparency & Explainability - Build and Validate - 3" } ] } }, "Smart Dubai (UAE)": { "AI Ethics Principles & Guidelines": { "link": "AI Ethics Principles & Guidelines", "requirements": [ { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.1.1.3" }, { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.1.2.1" }, { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.1.2.2" }, { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.1.3.1" }, { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.1.5.1" }, { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.1.5.2" }, { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.2.2.5" }, { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.2.2.6" }, { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.2.4.6" }, { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.2.4.7" }, { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.2.6.1" }, { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.4.1.2" }, { "requirementID": "https://www.digitaldubai.ae/docs/default-source/ai-principles-resources/ai-ethics.pdf", "requirementText": "1.4.2.2" } ] } }, "U.S. Department of Health & Human Services": { "Trustworthy AI (TAI) Playbook: Executive Summary": { "link": "Trustworthy AI (TAI) Playbook: Executive Summary", "requirements": [ { "requirementID": "https://www.hhs.gov/sites/default/files/hhs-trustworthy-ai-playbook-executive-summary.pdf", "requirementText": "Fair / Impartial" } ] } }, "UAE Ministry of Cabinet Affairs": { "The UAE Charter for the Development and Use of Artificial Intelligence": { "link": "The UAE Charter for the Development and Use of Artificial Intelligence", "requirements": [ { "requirementID": "https://uaelegislation.gov.ae/en/policy/details/the-uae-charter-for-the-development-and-use-of-artificial-intelligence#:~:text=The%20charter%20covers%20the%20following%20priorities%20and,and%20use%20of%20AI%20in%20the%20country.", "requirementText": "1. Strengthening Human-Machine Ties" }, { "requirementID": "https://uaelegislation.gov.ae/en/policy/details/the-uae-charter-for-the-development-and-use-of-artificial-intelligence#:~:text=The%20charter%20covers%20the%20following%20priorities%20and,and%20use%20of%20AI%20in%20the%20country.", "requirementText": "2. Safety" }, { "requirementID": "https://uaelegislation.gov.ae/en/policy/details/the-uae-charter-for-the-development-and-use-of-artificial-intelligence#:~:text=The%20charter%20covers%20the%20following%20priorities%20and,and%20use%20of%20AI%20in%20the%20country.", "requirementText": "3. Algorithmic Bias" }, { "requirementID": "https://uaelegislation.gov.ae/en/policy/details/the-uae-charter-for-the-development-and-use-of-artificial-intelligence#:~:text=The%20charter%20covers%20the%20following%20priorities%20and,and%20use%20of%20AI%20in%20the%20country.", "requirementText": "8. Technological Excellence" }, { "requirementID": "https://uaelegislation.gov.ae/en/policy/details/the-uae-charter-for-the-development-and-use-of-artificial-intelligence#:~:text=The%20charter%20covers%20the%20following%20priorities%20and,and%20use%20of%20AI%20in%20the%20country.", "requirementText": "9. Human Commitment" }, { "requirementID": "https://uaelegislation.gov.ae/en/policy/details/the-uae-charter-for-the-development-and-use-of-artificial-intelligence#:~:text=The%20charter%20covers%20the%20following%20priorities%20and,and%20use%20of%20AI%20in%20the%20country.", "requirementText": "10. Peaceful Coexistence with AI" }, { "requirementID": "https://uaelegislation.gov.ae/en/policy/details/the-uae-charter-for-the-development-and-use-of-artificial-intelligence#:~:text=The%20charter%20covers%20the%20following%20priorities%20and,and%20use%20of%20AI%20in%20the%20country.", "requirementText": "11. Promoting AI Awareness for an Inclusive Future" }, { "requirementID": "https://uaelegislation.gov.ae/en/policy/details/the-uae-charter-for-the-development-and-use-of-artificial-intelligence#:~:text=The%20charter%20covers%20the%20following%20priorities%20and,and%20use%20of%20AI%20in%20the%20country.", "requirementText": "12. Commitment to Treaties and Applicable Laws" } ] } } } } }