{"auditId":"aud_01KS70FB8DCPW4MD6ZHCSK","status":"completed","startedAt":"2026-05-22T04:53:06.445Z","updatedAt":"2026-05-22T04:53:39.937Z","progressNote":"Risk class: high — running clause checkers","report":{"auditId":"aud_01KS70FB8DCPW4MD6ZHCSK","source":{"kind":"repo","owner":"gpt-engineer-org","repo":"gpt-engineer"},"input":{"auditId":"aud_01KS70FB8DCPW4MD6ZHCSK","source":{"kind":"repo","owner":"gpt-engineer-org","repo":"gpt-engineer"},"repoUrl":"https://github.com/gpt-engineer-org/gpt-engineer","regulations":["eu-ai-act-2024-08","nist-ai-rmf-1.0","iso-42001-2023","gdpr-2016"]},"fetch":{"repoUrl":"https://github.com/gpt-engineer-org/gpt-engineer","owner":"gpt-engineer-org","repo":"gpt-engineer","ref":"a90fcd543eedcc0ff2c34561bc0785d2ba83c47e","commitSha":"a90fcd543eedcc0ff2c34561bc0785d2ba83c47e","worktreeDir":"/tmp/ai-auditor/aud_01KS70FB8DCPW4MD6ZHCSK","fileCount":182,"totalBytes":16764636,"languages":{"Markdown":52353,"YAML":7191,"Shell":517,"Python":352284,"HTML":954,"JSON":179380},"inventory":{"hasReadme":true,"hasLicense":true,"hasModelCard":false,"hasDataCard":false,"hasRiskRegister":false,"hasThreatModel":false,"hasSecurityMd":false,"hasPrivacyMd":false,"hasCodeOfConduct":true,"hasContributing":true,"hasCiWorkflows":true,"hasEvalDir":true,"hasTestsDir":true,"governanceDocFiles":[".github/CODE_OF_CONDUCT.md",".github/CONTRIBUTING.md","LICENSE","README.md","docker/README.md","docs/examples/open_llms/README.md","projects/example-improve/README.md"],"configFiles":["pyproject.toml"]},"fetchedAt":"2026-05-22T04:53:08.981Z"},"recon":{"signals":{"agent_framework":{"signal":"agent_framework","fired":true,"strength":0.9659893391883272,"evidence":[{"file":"pyproject.toml","rule":"manifest_framework_dep","detail":"LangChain"},{"file":"pyproject.toml","rule":"manifest_framework_dep","detail":"Anthropic SDK"},{"file":"docs/examples/open_llms/langchain_interface.py","lines":[3,3],"snippet":"import os from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler from langchain_openai import ChatOpenAI","rule":"langchain_import"},{"file":"docs/examples/open_llms/langchain_interface.py","lines":[4,4],"snippet":"dOutCallbackHandler from langchain_openai import ChatOpenAI model = ChatOpenAI( model=os.getenv(\"MODEL_NAME\"), temperature=0.1,","rule":"langchain_import"},{"file":"docs/examples/open_llms/openai_api_interface.py","lines":[5,5],"snippet":"rt OpenAI client = OpenAI( base_url=os.getenv(\"OPENAI_API_BASE\"), api_key=os.getenv(\"OPENAI_API_KEY\") ) response = client.chat.complet","rule":"openai_sdk_use"},{"file":"gpt_engineer/applications/cli/main.py","lines":[42,42],"snippet":"import load_dotenv from langchain.globals import set_llm_cache from langchain_community.cache import SQLiteCache from termcolor import colo","rule":"langchain_import"},{"file":"gpt_engineer/applications/cli/main.py","lines":[43,43],"snippet":"mport set_llm_cache from langchain_community.cache import SQLiteCache from termcolor import colored from gpt_engineer.applications.cli.cli_","rule":"langchain_import"},{"file":"gpt_engineer/benchmark/__main__.py","lines":[30,30],"snippet":"onal import typer from langchain.globals import set_llm_cache from langchain_community.cache import SQLiteCache from gpt_engineer.applica","rule":"langchain_import"},{"file":"gpt_engineer/benchmark/__main__.py","lines":[31,31],"snippet":"mport set_llm_cache from langchain_community.cache import SQLiteCache from gpt_engineer.applications.cli.main import load_env_if_needed fro","rule":"langchain_import"},{"file":"gpt_engineer/core/ai.py","lines":[29,29],"snippet":"i import pyperclip from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler from langchain.chat_models.base import B","rule":"langchain_import"},{"file":"gpt_engineer/core/ai.py","lines":[30,30],"snippet":"dOutCallbackHandler from langchain.chat_models.base import BaseChatModel from langchain.schema import ( AIMessage, HumanMessage,","rule":"langchain_import"},{"file":"gpt_engineer/core/ai.py","lines":[31,31],"snippet":"mport BaseChatModel from langchain.schema import ( AIMessage, HumanMessage, SystemMessage, messages_from_dict, messages_","rule":"langchain_import"},{"file":"gpt_engineer/core/ai.py","lines":[38,38],"snippet":"messages_to_dict, ) from langchain_anthropic import ChatAnthropic from langchain_openai import AzureChatOpenAI, ChatOpenAI from gpt_enginee","rule":"langchain_import"},{"file":"gpt_engineer/core/ai.py","lines":[39,39],"snippet":"mport ChatAnthropic from langchain_openai import AzureChatOpenAI, ChatOpenAI from gpt_engineer.core.token_usage import TokenUsageLog # Typ","rule":"langchain_import"},{"file":"gpt_engineer/core/default/steps.py","lines":[42,42],"snippet":"ableMapping, Union from langchain.schema import HumanMessage, SystemMessage from termcolor import colored from gpt_engineer.core.ai import","rule":"langchain_import"},{"file":"gpt_engineer/core/token_usage.py","lines":[11,11],"snippet":"n import tiktoken from langchain.schema import AIMessage, HumanMessage, SystemMessage from PIL import Image # workaround for function mov","rule":"langchain_import"},{"file":"gpt_engineer/core/token_usage.py","lines":[17,17],"snippet":"ai_info.py try: from langchain.callbacks.openai_info import ( get_openai_token_cost_for_model, # fmt: skip ) except ImportE","rule":"langchain_import"},{"file":"gpt_engineer/core/token_usage.py","lines":[21,21],"snippet":"pt ImportError: from langchain_community.callbacks.openai_info import ( get_openai_token_cost_for_model, # fmt: skip ) Me","rule":"langchain_import"},{"file":"gpt_engineer/tools/custom_steps.py","lines":[5,5],"snippet":"import List, Union from langchain.schema import AIMessage, HumanMessage, SystemMessage from gpt_engineer.core.ai import AI from gpt_engine","rule":"langchain_import"},{"file":"tests/applications/cli/test_cli_agent.py","lines":[6,6],"snippet":"ile import pytest from langchain.schema import AIMessage from gpt_engineer.applications.cli.cli_agent import CliAgent from gpt_engineer.c","rule":"langchain_import"},{"file":"tests/core/default/test_simple_agent.py","lines":[5,5],"snippet":"ile import pytest from langchain.schema import AIMessage from gpt_engineer.core.default.disk_execution_env import DiskExecutionEnv from g","rule":"langchain_import"},{"file":"tests/core/default/test_steps.py","lines":[8,8],"snippet":"ock import pytest from langchain.schema import SystemMessage from gpt_engineer.core.ai import AI from gpt_engineer.core.default.disk_memo","rule":"langchain_import"},{"file":"tests/core/test_ai.py","lines":[1,1],"snippet":"from langchain.chat_models.base import BaseChatModel from langchain_community.chat_models.fake import FakeListChatModel","rule":"langchain_import"},{"file":"tests/core/test_ai.py","lines":[2,2],"snippet":"mport BaseChatModel from langchain_community.chat_models.fake import FakeListChatModel from gpt_engineer.core.ai import AI def mock_creat","rule":"langchain_import"},{"file":"tests/core/test_salvage_correct_hunks.py","lines":[8,8],"snippet":"ist import pytest from langchain_core.messages import AIMessage from gpt_engineer.core.default.disk_memory import DiskMemory from gpt_eng","rule":"langchain_import"}]},"model_usage":{"signal":"model_usage","fired":true,"strength":0.8455773409227347,"evidence":[{"file":"gpt_engineer/applications/cli/main.py","lines":[284,284],"snippet":"n.get(\"MODEL_NAME\", \"gpt-4o\"), \"--model\", \"-m\", help=\"model id string\" ), temperature: float = typer.Option( 0.1, \"-","rule":"model_string_openai","detail":"gpt-4o"},{"file":"gpt_engineer/core/ai.py","lines":[90,90],"snippet":"model_name=\"gpt-4-turbo\", temperature=0.1, azure_endpoint=None, streaming=True, vision=False, )","rule":"model_string_openai","detail":"gpt-4-turbo"},{"file":"gpt_engineer/core/ai.py","lines":[102,102],"snippet":"to use, by default \"gpt-4\". temperature : float, optional The temperature to use for the model, by default 0.1.","rule":"model_string_openai","detail":"gpt-4"},{"file":"gpt_engineer/core/ai.py","lines":[112,112],"snippet":"me) or (\"gpt-4-turbo\" in model_name and \"preview\" not in model_name) or (\"claude\" in model_name) ) s","rule":"model_string_openai","detail":"gpt-4-turbo"},{"file":"gpt_engineer/core/token_usage.py","lines":[76,76],"snippet":"ame) if \"gpt-4\" in model_name or \"gpt-3.5\" in model_name else tiktoken.get_encoding(\"cl100k_base\") ) de","rule":"model_string_openai","detail":"gpt-4"},{"file":"gpt_engineer/core/token_usage.py","lines":[76,76],"snippet":"4\" in model_name or \"gpt-3.5\" in model_name else tiktoken.get_encoding(\"cl100k_base\") ) def num_tokens(self, txt: s","rule":"model_string_openai","detail":"gpt-3.5"},{"file":"tests/core/test_ai.py","lines":[14,14],"snippet":"model) ai = AI(\"gpt-4\") # act response_messages = ai.start(\"system prompt\", \"user prompt\", step_name=\"step name\") # asser","rule":"model_string_openai","detail":"gpt-4"},{"file":"tests/core/test_ai.py","lines":[27,27],"snippet":"model) ai = AI(\"gpt-4\") response_messages = ai.start(\"system prompt\", \"user prompt\", step_name=\"step name\") # act response","rule":"model_string_openai","detail":"gpt-4"},{"file":"tests/core/test_ai.py","lines":[43,43],"snippet":"model) ai = AI(\"gpt-4\") # act response_messages = ai.start(\"system prompt\", \"user prompt\", step_name=\"step name\") usageCos","rule":"model_string_openai","detail":"gpt-4"},{"file":"tests/core/test_token_usage.py","lines":[17,17],"snippet":"log = TokenUsageLog(\"gpt-4\") request_messages = [ SystemMessage(content=\"my system message\"), HumanMessage(content=\"my u","rule":"model_string_openai","detail":"gpt-4"},{"file":"tests/core/test_token_usage.py","lines":[39,39],"snippet":"log = TokenUsageLog(\"gpt-4\") request_messages = [ SystemMessage(content=\"my system message\"), HumanMessage(content=\"my u","rule":"model_string_openai","detail":"gpt-4"},{"file":"tests/core/test_token_usage.py","lines":[57,57],"snippet":"age_log = Tokenizer(\"gpt-4\") image_path = Path(__file__).parent.parent / \"test_data\" / \"mona_lisa.jpg\" # Check if the image file exi","rule":"model_string_openai","detail":"gpt-4"},{"file":"tests/core/test_token_usage.py","lines":[82,82],"snippet":"log = TokenUsageLog(\"gpt-4\") request_messages = [ SystemMessage(content=\"My system message\"), HumanMessage(","rule":"model_string_openai","detail":"gpt-4"}],"meta":{"models":{}}},"logging_hooks":{"signal":"logging_hooks","fired":true,"strength":0.8,"evidence":[{"file":"gpt_engineer/applications/cli/main.py","lines":[30,30],"snippet":"difflib import json import logging import os import platform import subprocess import sys from pathlib import Path import openai import ty","rule":"import_logging"},{"file":"gpt_engineer/core/ai.py","lines":[19,19],"snippet":"ations import json import logging import os from pathlib import Path from typing import Any, List, Optional, Union import backoff import","rule":"import_logging"},{"file":"gpt_engineer/core/ai.py","lines":[118,118],"snippet":"odel_name) logger.debug(f\"Using model {self.model_name}\") def start(self, system: str, user: Any, *, step_name: str) -> List[M","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/core/ai.py","lines":[235,235],"snippet":"t=prompt)) logger.debug( \"Creating a new chat completion: %s\", \"\\n\".join([m.pretty_repr() for m in messages","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/core/ai.py","lines":[249,249],"snippet":"d(response) logger.debug(f\"Chat completion finished: {messages}\") return messages @backoff.on_exception(backoff.expo,","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/core/ai.py","lines":[421,421],"snippet":"t=prompt)) logger.debug(f\"Creating a new chat completion: {messages}\") msgs = self.serialize_messages(messages) py","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/core/ai.py","lines":[435,435],"snippet":"=response)) logger.debug(f\"Chat completion finished: {messages}\") return messages","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/core/chat_to_files.py","lines":[24,24],"snippet":"t transcripts. \"\"\" import logging import re from typing import Dict, Tuple from regex import regex from gpt_engineer.core.diff import AD","rule":"import_logging"},{"file":"gpt_engineer/core/diff.py","lines":[34,34],"snippet":"code changes. \"\"\" import logging from collections import Counter from typing import List RETAIN = \"retain\" ADD = \"add\" REMOVE = \"remove\"","rule":"import_logging"},{"file":"gpt_engineer/core/token_usage.py","lines":[3,3],"snippet":"rt base64 import io import logging import math from dataclasses import dataclass from typing import List, Union import tiktoken from lang","rule":"import_logging"}]},"oversight_hooks":{"signal":"oversight_hooks","fired":true,"strength":0.8921795685513096,"evidence":[{"file":"gpt_engineer/core/ai.py","lines":[33,33],"snippet":"AIMessage, HumanMessage, SystemMessage, messages_from_dict, messages_to_dict, ) from langchain_anthropic import ChatAnt","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[44,44],"snippet":"= Union[AIMessage, HumanMessage, SystemMessage] # Set up logging logger = logging.getLogger(__name__) class AI: \"\"\" A class that","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[141,141],"snippet":"ystem), HumanMessage(content=user), ] return self.next(messages, step_name=step_name) def _extract_content(","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[233,233],"snippet":"messages.append(HumanMessage(content=prompt)) logger.debug( \"Creating a new chat completion: %s\", \"\\n\".","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[284,284],"snippet":"e(content=\"Hello\"), HumanMessage(content=\"How's the weather?\")] >>> response = backoff_inference(messages) \"\"\" retur","rule":"human_in_loop"},{"file":"gpt_engineer/core/default/steps.py","lines":[42,42],"snippet":"chain.schema import HumanMessage, SystemMessage from termcolor import colored from gpt_engineer.core.ai import AI from gpt_engineer.core.ba","rule":"human_in_loop"},{"file":"gpt_engineer/core/default/steps.py","lines":[306,306],"snippet":"messages.append(HumanMessage(content=f\"{files_dict.to_chat()}\")) messages.append(HumanMessage(content=prompt.to_langchain_content())","rule":"human_in_loop"},{"file":"gpt_engineer/core/default/steps.py","lines":[307,307],"snippet":"messages.append(HumanMessage(content=prompt.to_langchain_content())) memory.log( DEBUG_LOG_FILE, \"UPLOADED FILES:\\n\"","rule":"human_in_loop"},{"file":"gpt_engineer/core/default/steps.py","lines":[326,326],"snippet":"append( HumanMessage( content=\"Some previously produced diffs were not on the requested format, or the code part","rule":"human_in_loop"},{"file":"gpt_engineer/core/token_usage.py","lines":[11,11],"snippet":"a import AIMessage, HumanMessage, SystemMessage from PIL import Image # workaround for function moved in: # https://github.com/langchain-ai","rule":"human_in_loop"},{"file":"gpt_engineer/core/token_usage.py","lines":[26,26],"snippet":"= Union[AIMessage, HumanMessage, SystemMessage] logger = logging.getLogger(__name__) @dataclass class TokenUsage: \"\"\" Dataclass","rule":"human_in_loop"},{"file":"gpt_engineer/tools/custom_steps.py","lines":[5,5],"snippet":"a import AIMessage, HumanMessage, SystemMessage from gpt_engineer.core.ai import AI from gpt_engineer.core.base_execution_env import BaseEx","rule":"human_in_loop"},{"file":"gpt_engineer/tools/custom_steps.py","lines":[18,18],"snippet":"= Union[AIMessage, HumanMessage, SystemMessage] MAX_SELF_HEAL_ATTEMPTS = 10 def get_platform_info() -> str: \"\"\" Returns a string","rule":"human_in_loop"},{"file":"tests/core/test_token_usage.py","lines":[9,9],"snippet":"chain.schema import HumanMessage, SystemMessage from PIL import Image from gpt_engineer.core.token_usage import Tokenizer, TokenUsageLog","rule":"human_in_loop"},{"file":"tests/core/test_token_usage.py","lines":[20,20],"snippet":"message\"), HumanMessage(content=\"my user prompt\"), ] response = \"response from model\" # act token_usage_log.update","rule":"human_in_loop"},{"file":"tests/core/test_token_usage.py","lines":[42,42],"snippet":"message\"), HumanMessage(content=\"my user prompt\"), ] response = \"response from model\" # act token_usage_log.update","rule":"human_in_loop"},{"file":"tests/core/test_token_usage.py","lines":[86,86],"snippet":"message\"), HumanMessage( content=[ {\"type\": \"text\", \"text\": \"My user message\"}, {","rule":"human_in_loop"}]},"eval_artefacts":{"signal":"eval_artefacts","fired":true,"strength":0.590848501887865,"evidence":[{"file":"evals/","rule":"eval_or_test_dir_present"},{"file":".github/workflows/","rule":"ci_workflows_present"},{"file":"pyproject.toml","rule":"manifest_eval_dep","detail":"pytest"}]},"governance_docs":{"signal":"governance_docs","fired":true,"strength":0.7380392160057028,"evidence":[{"file":".github/CODE_OF_CONDUCT.md","rule":"governance_doc_present"},{"file":".github/CONTRIBUTING.md","rule":"governance_doc_present"},{"file":"LICENSE","rule":"governance_doc_present"},{"file":"README.md","rule":"governance_doc_present"},{"file":"docker/README.md","rule":"governance_doc_present"},{"file":"docs/examples/open_llms/README.md","rule":"governance_doc_present"},{"file":"projects/example-improve/README.md","rule":"governance_doc_present"}]},"cybersec_hooks":{"signal":"cybersec_hooks","fired":true,"strength":0.590848501887865,"evidence":[{"file":"gpt_engineer/core/ai.py","lines":[253,253],"snippet":"ackoff.expo, openai.RateLimitError, max_tries=7, max_time=45) def backoff_inference(self, messages): \"\"\" Perform inferen","rule":"rate_limit_usage"},{"file":"gpt_engineer/core/ai.py","lines":[278,278],"snippet":"openai.error.RateLimitError If the number of retries exceeds the maximum or if the rate limit persists beyond the","rule":"rate_limit_usage"},{"file":"gpt_engineer/core/ai.py","lines":[280,280],"snippet":"ultimately raise a RateLimitError. Example ------- >>> messages = [SystemMessage(content=\"Hello\"), HumanMessage(co","rule":"rate_limit_usage"}]},"provenance_hooks":{"signal":"provenance_hooks","fired":false,"strength":0,"evidence":[]},"data_io":{"signal":"data_io","fired":true,"strength":0.4,"evidence":[{"file":"scripts/test_api.py","lines":[25,25],"snippet":"\"\"\" response = requests.post(url, json=extra_arguments) return response if __name__ == \"__main__\": URL_BASE = \"http://127.0.0","rule":"http_external_io"}]},"pii_signals":{"signal":"pii_signals","fired":false,"strength":0,"evidence":[]},"bio_health_signals":{"signal":"bio_health_signals","fired":false,"strength":0,"evidence":[]},"employment_signals":{"signal":"employment_signals","fired":true,"strength":0.4,"evidence":[{"file":"gpt_engineer/core/diff.py","lines":[190,190],"snippet":"e is that, for each candidate check match to the next line etc (recursively) start_ind = list(pot_start_lines.keys())[","rule":"employment_terms"}]},"bio_metric_signals":{"signal":"bio_metric_signals","fired":false,"strength":0,"evidence":[]},"education_signals":{"signal":"education_signals","fired":false,"strength":0,"evidence":[]},"law_enforcement_signals":{"signal":"law_enforcement_signals","fired":false,"strength":0,"evidence":[]},"migration_signals":{"signal":"migration_signals","fired":false,"strength":0,"evidence":[]},"justice_signals":{"signal":"justice_signals","fired":false,"strength":0,"evidence":[]},"essential_services_signals":{"signal":"essential_services_signals","fired":false,"strength":0,"evidence":[]},"critical_infra_signals":{"signal":"critical_infra_signals","fired":false,"strength":0,"evidence":[]},"content_generation_signals":{"signal":"content_generation_signals","fired":false,"strength":0,"evidence":[]},"emotion_recognition_signals":{"signal":"emotion_recognition_signals","fired":false,"strength":0,"evidence":[]}}},"scope":{"regulationIds":["eu-ai-act-2024-08","nist-ai-rmf-1.0","iso-42001-2023","gdpr-2016"],"notes":["Auditing against packs: eu-ai-act-2024-08, nist-ai-rmf-1.0, iso-42001-2023, gdpr-2016"]},"map":{"classification":"high","annexIiiCategories":["4"],"art50Triggers":["50(1)"],"drivingSignals":[{"signal":"employment_signals","category":"4"}],"rationale":["Classified HIGH-RISK based on 1 Annex III signal(s).","Annex III §4 (Employment, workers management, access to self-employment) — signal \"employment_signals\" fired with strength 0.40","Art 50 50(1) (Interaction with natural persons) — signal \"agent_framework\" fired"]},"checks":[{"clauseId":"eu-ai-act/art-5/1a-subliminal","regulationId":"eu-ai-act-2024-08","article":"5(1)(a)","title":"Subliminal techniques distorting behaviour","principle":"transparency","verdict":"pass","score":4,"rawScore":0,"evidence":[],"rules":[{"rule":"detect_manipulative_prompt_patterns","matched":0,"weight":1}],"rationale":"Composite raw score 0.00 (0/1 rules matched).","confidence":0.55,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-5/1b-vulnerability","regulationId":"eu-ai-act-2024-08","article":"5(1)(b)","title":"Exploiting vulnerabilities (age, disability, socio-economic)","principle":"fairness","verdict":"pass","score":4,"rawScore":0,"evidence":[],"rules":[{"rule":"detect_protected_attribute_targeting","matched":0,"weight":1}],"rationale":"Composite raw score 0.00 (0/1 rules matched).","confidence":0.55,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-5/1c-social-scoring","regulationId":"eu-ai-act-2024-08","article":"5(1)(c)","title":"Social scoring leading to detrimental treatment","principle":"fairness","verdict":"pass","score":4,"rawScore":0,"evidence":[],"rules":[{"rule":"detect_scoring_with_persistent_user_state","matched":0,"weight":1}],"rationale":"Composite raw score 0.00 (0/1 rules matched). Supporting docs may exist outside the repo.","confidence":0.55,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-5/1d-predictive-policing","regulationId":"eu-ai-act-2024-08","article":"5(1)(d)","title":"Predictive policing solely from profiling","principle":"fairness","verdict":"pass","score":4,"rawScore":0,"evidence":[],"rules":[{"rule":"detect_crime_risk_scoring_from_profile","matched":0,"weight":1}],"rationale":"Composite raw score 0.00 (0/1 rules matched).","confidence":0.55,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-5/1e-facial-scraping","regulationId":"eu-ai-act-2024-08","article":"5(1)(e)","title":"Untargeted facial image scraping for face databases","principle":"privacy","verdict":"pass","score":4,"rawScore":0,"evidence":[],"rules":[{"rule":"detect_face_image_scraping","matched":0,"weight":1}],"rationale":"Composite raw score 0.00 (0/1 rules matched).","confidence":0.55,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-5/1f-emotion-workplace-edu","regulationId":"eu-ai-act-2024-08","article":"5(1)(f)","title":"Emotion recognition in workplace and education","principle":"privacy","verdict":"pass","score":4,"rawScore":0,"evidence":[],"rules":[{"rule":"detect_emotion_recognition_in_workplace_education","matched":0,"weight":1}],"rationale":"Composite raw score 0.00 (0/1 rules matched).","confidence":0.55,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-5/1g-biometric-categorisation","regulationId":"eu-ai-act-2024-08","article":"5(1)(g)","title":"Biometric categorisation by protected attributes","principle":"fairness","verdict":"pass","score":4,"rawScore":0,"evidence":[],"rules":[{"rule":"detect_biometric_categorisation_by_protected_attrs","matched":0,"weight":1}],"rationale":"Composite raw score 0.00 (0/1 rules matched).","confidence":0.55,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-5/1h-realtime-rbi","regulationId":"eu-ai-act-2024-08","article":"5(1)(h)","title":"Real-time remote biometric identification in public spaces","principle":"privacy","verdict":"external","score":null,"rawScore":null,"evidence":[],"rules":[],"rationale":"Deployment context (public space, real-time, law enforcement use, judicial\nauthorisation) is operational, not knowable from code. Always external.","confidence":1,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-9/risk-management","regulationId":"eu-ai-act-2024-08","article":"9","title":"Risk management system established, implemented, documented","principle":"safety","verdict":"partial","score":2,"rawScore":0.3,"evidence":[{"file":".github/workflows/ci.yaml","rule":"ci_eval_gates"}],"rules":[{"rule":"presence_of_risk_register","matched":0,"weight":0.4},{"rule":"presence_of_threat_model","matched":0,"weight":0.3},{"rule":"ci_eval_gates","matched":1,"weight":0.3}],"rationale":"Composite raw score 0.30 (1/3 rules matched). Supporting docs may exist outside the repo.\n\nLLM judge (confidence 0.35): CI/CD evaluation gates provide some risk control mechanism, but absence of documented risk register and threat model leaves critical identification, analysis, and systematic review requirements unmet. Human review of external documentation (design files, risk assessments, governance records) is needed to determine full compliance with the continuous iterative risk management lifecycle.","confidence":0.35,"needsLlmJudge":true,"verified":true,"verifyMethod":"llm-judge"},{"clauseId":"eu-ai-act/art-10/data-governance","regulationId":"eu-ai-act-2024-08","article":"10","title":"Data and data governance practices documented","principle":"privacy","verdict":"fail","score":0,"rawScore":0.1,"evidence":[{"file":"scripts/test_api.py","lines":[25,25],"snippet":"\"\"\" response = requests.post(url, json=extra_arguments) return response if __name__ == \"__main__\": URL_BASE = \"http://127.0.0","rule":"http_external_io"}],"rules":[{"rule":"presence_of_data_card","matched":0,"weight":0.5},{"rule":"data_loading_code_quality","matched":0.4,"weight":0.25},{"rule":"bias_evaluation_present","matched":0,"weight":0.25}],"rationale":"Composite raw score 0.10 (0/3 rules matched). Supporting docs may exist outside the repo.","confidence":0.75,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-11/technical-documentation","regulationId":"eu-ai-act-2024-08","article":"11","title":"Technical documentation drawn up before placing on market","principle":"auditability","verdict":"fail","score":0,"rawScore":0.075,"evidence":[{"file":"README.md","rule":"readme_quality","detail":"1 required sections present"}],"rules":[{"rule":"presence_of_model_card","matched":0,"weight":0.5},{"rule":"readme_quality","matched":0.25,"weight":0.3},{"rule":"architecture_docs","matched":0,"weight":0.2}],"rationale":"Composite raw score 0.07 (0/3 rules matched). Supporting docs may exist outside the repo.","confidence":0.725,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-12/p1-automatic-logs","regulationId":"eu-ai-act-2024-08","article":"12(1)","title":"Automatic recording of events over the lifetime","principle":"auditability","verdict":"partial","score":2,"rawScore":0.48000000000000004,"evidence":[{"file":"gpt_engineer/core/ai.py","lines":[118,118],"snippet":"odel_name) logger.debug(f\"Using model {self.model_name}\") def start(self, system: str, user: Any, *, step_name: str) -> List[M","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/core/ai.py","lines":[235,235],"snippet":"t=prompt)) logger.debug( \"Creating a new chat completion: %s\", \"\\n\".join([m.pretty_repr() for m in messages","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/core/ai.py","lines":[249,249],"snippet":"d(response) logger.debug(f\"Chat completion finished: {messages}\") return messages @backoff.on_exception(backoff.expo,","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/core/ai.py","lines":[421,421],"snippet":"t=prompt)) logger.debug(f\"Creating a new chat completion: {messages}\") msgs = self.serialize_messages(messages) py","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/core/ai.py","lines":[435,435],"snippet":"=response)) logger.debug(f\"Chat completion finished: {messages}\") return messages","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/applications/cli/main.py","lines":[30,30],"snippet":"difflib import json import logging import os import platform import subprocess import sys from pathlib import Path import openai import ty","rule":"import_logging"},{"file":"gpt_engineer/core/ai.py","lines":[19,19],"snippet":"ations import json import logging import os from pathlib import Path from typing import Any, List, Optional, Union import backoff import","rule":"import_logging"}],"rules":[{"rule":"structured_logging_imported","matched":0,"weight":0.3},{"rule":"logging_at_tool_call_boundaries","matched":0.8,"weight":0.5},{"rule":"logging_persistent_sink","matched":0.4,"weight":0.2}],"rationale":"Composite raw score 0.48 (1/3 rules matched).\n\nLLM judge (confidence 0.62): The system demonstrates logging at tool call boundaries (5 evidence points in ai.py showing logger.debug calls around chat completion events) and imports the logging module, but lacks evidence of structured logging framework and persistent storage sink configuration. Automatic recording occurs at critical points but completeness across system lifetime and durability guarantees remain unverified.","confidence":0.62,"needsLlmJudge":true,"verified":true,"verifyMethod":"llm-judge"},{"clauseId":"eu-ai-act/art-12/p2-traceability","regulationId":"eu-ai-act-2024-08","article":"12(2)","title":"Logging ensures traceability appropriate to risk","principle":"auditability","verdict":"partial","score":2,"rawScore":0.575,"evidence":[{"file":"gpt_engineer/applications/cli/main.py","lines":[30,30],"snippet":"difflib import json import logging import os import platform import subprocess import sys from pathlib import Path import openai import ty","rule":"import_logging"},{"file":"gpt_engineer/core/ai.py","lines":[19,19],"snippet":"ations import json import logging import os from pathlib import Path from typing import Any, List, Optional, Union import backoff import","rule":"import_logging"},{"file":"gpt_engineer/core/ai.py","lines":[118,118],"snippet":"odel_name) logger.debug(f\"Using model {self.model_name}\") def start(self, system: str, user: Any, *, step_name: str) -> List[M","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/core/ai.py","lines":[235,235],"snippet":"t=prompt)) logger.debug( \"Creating a new chat completion: %s\", \"\\n\".join([m.pretty_repr() for m in messages","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/core/ai.py","lines":[249,249],"snippet":"d(response) logger.debug(f\"Chat completion finished: {messages}\") return messages @backoff.on_exception(backoff.expo,","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/core/ai.py","lines":[421,421],"snippet":"t=prompt)) logger.debug(f\"Creating a new chat completion: {messages}\") msgs = self.serialize_messages(messages) py","rule":"log_at_tool_boundary"},{"file":"gpt_engineer/applications/cli/main.py","lines":[30,30],"snippet":"difflib import json import logging import os import platform import subprocess import sys from pathlib import Path import openai import ty","rule":"import_logging"},{"file":"gpt_engineer/core/ai.py","lines":[19,19],"snippet":"ations import json import logging import os from pathlib import Path from typing import Any, List, Optional, Union import backoff import","rule":"import_logging"}],"rules":[{"rule":"logs_include_input_output_pairs","matched":0.8,"weight":0.5},{"rule":"logs_include_model_identity","matched":0.7,"weight":0.25},{"rule":"logs_include_request_id","matched":0,"weight":0.25}],"rationale":"Composite raw score 0.57 (2/3 rules matched). Supporting docs may exist outside the repo.","confidence":1,"needsLlmJudge":true,"verified":true,"verifyMethod":"skipped"},{"clauseId":"eu-ai-act/art-13/deployer-instructions","regulationId":"eu-ai-act-2024-08","article":"13","title":"Transparent operation and instructions for use","principle":"transparency","verdict":"fail","score":0,"rawScore":0.125,"evidence":[{"file":"README.md","rule":"readme_quality","detail":"1 required sections present"},{"file":"README.md","rule":"output_interpretation_guidance"},{"file":"README.md","rule":"limitations_section_present"}],"rules":[{"rule":"presence_of_deployer_instructions","matched":0.25,"weight":0.5},{"rule":"output_interpretation_guidance","matched":0,"weight":0.3},{"rule":"limitations_section_present","matched":0,"weight":0.2}],"rationale":"Composite raw score 0.13 (0/3 rules matched). Supporting docs may exist outside the repo.","confidence":0.775,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-14/p1-oversight-measures","regulationId":"eu-ai-act-2024-08","article":"14(1)","title":"Effective human oversight designed and built-in","principle":"human-oversight","verdict":"fail","score":0,"rawScore":0.4460897842756548,"evidence":[{"file":"gpt_engineer/core/ai.py","lines":[33,33],"snippet":"AIMessage, HumanMessage, SystemMessage, messages_from_dict, messages_to_dict, ) from langchain_anthropic import ChatAnt","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[44,44],"snippet":"= Union[AIMessage, HumanMessage, SystemMessage] # Set up logging logger = logging.getLogger(__name__) class AI: \"\"\" A class that","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[141,141],"snippet":"ystem), HumanMessage(content=user), ] return self.next(messages, step_name=step_name) def _extract_content(","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[233,233],"snippet":"messages.append(HumanMessage(content=prompt)) logger.debug( \"Creating a new chat completion: %s\", \"\\n\".","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[284,284],"snippet":"e(content=\"Hello\"), HumanMessage(content=\"How's the weather?\")] >>> response = backoff_inference(messages) \"\"\" retur","rule":"human_in_loop"},{"file":"gpt_engineer/core/default/steps.py","lines":[42,42],"snippet":"chain.schema import HumanMessage, SystemMessage from termcolor import colored from gpt_engineer.core.ai import AI from gpt_engineer.core.ba","rule":"human_in_loop"}],"rules":[{"rule":"human_in_loop_hooks_present","matched":0.8921795685513096,"weight":0.5},{"rule":"oversight_ui_present","matched":0,"weight":0.3},{"rule":"tool_calls_have_dry_run","matched":0,"weight":0.2}],"rationale":"Composite raw score 0.45 (1/3 rules matched).\n\nLLM judge (confidence 0.85): While human-machine message handling is present (HumanMessage/AIMessage structures), the evidence shows no actual oversight UI, dry-run capabilities, or documented mechanisms for humans to effectively intervene during system operation. Message passing alone is insufficient for 'effective oversight' as required by Art. 14(1); the system lacks the control interface tools necessary for active human supervision.","confidence":0.85,"needsLlmJudge":true,"verified":true,"verifyMethod":"llm-judge"},{"clauseId":"eu-ai-act/art-14/p4d-stop-button","regulationId":"eu-ai-act-2024-08","article":"14(4)(d)","title":"Interrupt / stop function reachable by overseer","principle":"human-oversight","verdict":"fail","score":0,"rawScore":0,"evidence":[],"rules":[{"rule":"kill_switch_present","matched":0,"weight":0.7},{"rule":"graceful_shutdown_handler","matched":0,"weight":0.3}],"rationale":"Composite raw score 0.00 (0/2 rules matched).","confidence":0.6,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-14/p4e-override","regulationId":"eu-ai-act-2024-08","article":"14(4)(e)","title":"Ability to override / reverse the system's output","principle":"human-oversight","verdict":"fail","score":0,"rawScore":0.6,"evidence":[{"file":"gpt_engineer/core/ai.py","lines":[33,33],"snippet":"AIMessage, HumanMessage, SystemMessage, messages_from_dict, messages_to_dict, ) from langchain_anthropic import ChatAnt","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[44,44],"snippet":"= Union[AIMessage, HumanMessage, SystemMessage] # Set up logging logger = logging.getLogger(__name__) class AI: \"\"\" A class that","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[141,141],"snippet":"ystem), HumanMessage(content=user), ] return self.next(messages, step_name=step_name) def _extract_content(","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[233,233],"snippet":"messages.append(HumanMessage(content=prompt)) logger.debug( \"Creating a new chat completion: %s\", \"\\n\".","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[284,284],"snippet":"e(content=\"Hello\"), HumanMessage(content=\"How's the weather?\")] >>> response = backoff_inference(messages) \"\"\" retur","rule":"human_in_loop"},{"file":"gpt_engineer/core/default/steps.py","lines":[42,42],"snippet":"chain.schema import HumanMessage, SystemMessage from termcolor import colored from gpt_engineer.core.ai import AI from gpt_engineer.core.ba","rule":"human_in_loop"}],"rules":[{"rule":"override_path_present","matched":1,"weight":0.6},{"rule":"decisions_are_addressable","matched":0,"weight":0.4}],"rationale":"Composite raw score 0.60 (1/2 rules matched).\n\nLLM judge (confidence 0.85): Evidence shows human-in-loop message passing infrastructure (override_path_present matched), but repository lacks concrete implementation of decision reversal mechanisms or UI/API endpoints enabling humans to actually override or reverse AI outputs. The matched rule alone (0.6 weight) is insufficient without addressable decision structures (0.4 weight) required by Article 14(4)(e).","confidence":0.85,"needsLlmJudge":true,"verified":true,"verifyMethod":"llm-judge"},{"clauseId":"eu-ai-act/art-15/p1-accuracy","regulationId":"eu-ai-act-2024-08","article":"15(1)","title":"Appropriate level of accuracy declared and tested","principle":"safety","verdict":"partial","score":2,"rawScore":0.7,"evidence":[{"file":"evals/","rule":"eval_suite_present"},{"file":"README.md","rule":"metrics_documented"},{"file":".github/workflows/ci.yaml","rule":"ci_eval_gates"}],"rules":[{"rule":"eval_suite_present","matched":1,"weight":0.5},{"rule":"metrics_documented","matched":0,"weight":0.3},{"rule":"eval_in_ci","matched":1,"weight":0.2}],"rationale":"Composite raw score 0.70 (2/3 rules matched).\n\nLLM judge (confidence 0.72): The system demonstrates evaluation infrastructure (eval_suite_present, eval_in_ci) but lacks documented metrics specifications required by Article 15(1). Without explicit accuracy, robustness, and cybersecurity metrics documentation, compliance with 'appropriate level' and 'consistent performance' requirements cannot be fully verified.","confidence":0.72,"needsLlmJudge":true,"verified":true,"verifyMethod":"llm-judge"},{"clauseId":"eu-ai-act/art-15/p4-robustness","regulationId":"eu-ai-act-2024-08","article":"15(4)","title":"Resilience to errors, faults, inconsistencies","principle":"safety","verdict":"fail","score":1,"rawScore":0.16000000000000003,"evidence":[{"file":"pyproject.toml","rule":"manifest_framework_dep","detail":"LangChain"},{"file":"pyproject.toml","rule":"manifest_framework_dep","detail":"Anthropic SDK"}],"rules":[{"rule":"error_handling_at_tool_boundaries","matched":0.4,"weight":0.4},{"rule":"retry_logic","matched":0,"weight":0.3},{"rule":"fallback_behaviour","matched":0,"weight":0.3}],"rationale":"Composite raw score 0.16 (0/3 rules matched).","confidence":0.81,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-15/p5-cybersecurity","regulationId":"eu-ai-act-2024-08","article":"15(5)","title":"Cybersecurity measures appropriate to circumstances","principle":"security-governance","verdict":"fail","score":0,"rawScore":0.118169700377573,"evidence":[{"file":"gpt_engineer/core/ai.py","lines":[253,253],"snippet":"ackoff.expo, openai.RateLimitError, max_tries=7, max_time=45) def backoff_inference(self, messages): \"\"\" Perform inferen","rule":"rate_limit_usage"},{"file":"gpt_engineer/core/ai.py","lines":[278,278],"snippet":"openai.error.RateLimitError If the number of retries exceeds the maximum or if the rate limit persists beyond the","rule":"rate_limit_usage"},{"file":"gpt_engineer/core/ai.py","lines":[280,280],"snippet":"ultimately raise a RateLimitError. Example ------- >>> messages = [SystemMessage(content=\"Hello\"), HumanMessage(co","rule":"rate_limit_usage"}],"rules":[{"rule":"prompt_injection_defences","matched":0,"weight":0.4},{"rule":"rate_limiting","matched":0.590848501887865,"weight":0.2},{"rule":"secrets_not_in_prompts","matched":0,"weight":0.2},{"rule":"adversarial_eval_present","matched":0,"weight":0.2}],"rationale":"Composite raw score 0.12 (1/4 rules matched).","confidence":0.818169700377573,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-26/p6-keep-logs","regulationId":"eu-ai-act-2024-08","article":"26(6)","title":"Deployer log-retention capability supported","principle":"accountability","verdict":"partial","score":2,"rawScore":0.5,"evidence":[{"file":"gpt_engineer/applications/cli/main.py","lines":[30,30],"snippet":"difflib import json import logging import os import platform import subprocess import sys from pathlib import Path import openai import ty","rule":"import_logging"},{"file":"gpt_engineer/core/ai.py","lines":[19,19],"snippet":"ations import json import logging import os from pathlib import Path from typing import Any, List, Optional, Union import backoff import","rule":"import_logging"}],"rules":[{"rule":"logs_externally_exportable","matched":0.5,"weight":1}],"rationale":"Composite raw score 0.50 (1/1 rules matched). Supporting docs may exist outside the repo.\n\nLLM judge (confidence 0.35): Logging imports are present, confirming technical capability for log generation, but evidence does not demonstrate: (1) automatic log generation by the AI system itself, (2) logs under deployer control, (3) retention policies meeting the six-month minimum, or (4) exported logs accessibility. Deterministic rule only confirms exportability potential; actual compliance requires documentation of retention procedures and log access mechanisms.","confidence":0.35,"needsLlmJudge":true,"verified":true,"verifyMethod":"llm-judge"},{"clauseId":"eu-ai-act/art-50/p1-ai-disclosure","regulationId":"eu-ai-act-2024-08","article":"50(1)","title":"Users informed they are interacting with an AI","principle":"transparency","verdict":"partial","score":2,"rawScore":0.4,"evidence":[{"file":"README.md","rule":"ai_disclosure_in_user_facing_strings"},{"file":"gpt_engineer/core/prompt.py","rule":"persona_not_human_impersonating"}],"rules":[{"rule":"ai_disclosure_in_user_facing_strings","matched":0,"weight":0.6},{"rule":"persona_not_human_impersonating","matched":1,"weight":0.4}],"rationale":"Composite raw score 0.40 (1/2 rules matched).\n\nLLM judge (confidence 0.45): The system fails explicit disclosure in user-facing strings (README.md) but demonstrates non-human persona design in core prompts, creating ambiguity about whether a reasonably informed user would recognize AI interaction. Human judgment needed on whether prompt-level design choices substitute for explicit disclosure statements.","confidence":0.45,"needsLlmJudge":true,"verified":true,"verifyMethod":"llm-judge"},{"clauseId":"eu-ai-act/art-50/p2-synthetic-content","regulationId":"eu-ai-act-2024-08","article":"50(2)","title":"AI-generated content marked as such, machine-readable","principle":"transparency","verdict":"n/a","score":null,"rawScore":null,"evidence":[],"rules":[],"rationale":"Clause subject matter not detected — required signals [content_generation_signals, provenance_hooks] did not fire in this repo.","confidence":1,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-50/p3-emotion-biometric-disclosure","regulationId":"eu-ai-act-2024-08","article":"50(3)","title":"Emotion recognition / biometric categorisation disclosure","principle":"transparency","verdict":"n/a","score":null,"rawScore":null,"evidence":[],"rules":[],"rationale":"Clause subject matter not detected — required signals [emotion_recognition_signals, bio_metric_signals] did not fire in this repo.","confidence":1,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"eu-ai-act/art-50/p4-deepfake-disclosure","regulationId":"eu-ai-act-2024-08","article":"50(4)","title":"Deepfake content labelled as artificially generated","principle":"transparency","verdict":"n/a","score":null,"rawScore":null,"evidence":[],"rules":[],"rationale":"Clause subject matter not detected — required signals [content_generation_signals] did not fire in this repo.","confidence":1,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"nist-ai-rmf/govern-1.4","regulationId":"nist-ai-rmf-1.0","article":"GOVERN 1.4","title":"Risk management process documented and accountable","principle":"accountability","verdict":"fail","score":0,"rawScore":0,"evidence":[],"rules":[{"rule":"presence_of_risk_register","matched":0,"weight":0.6},{"rule":"risk_owner_assignment","matched":0,"weight":0.4}],"rationale":"Composite raw score 0.00 (0/2 rules matched). Supporting docs may exist outside the repo.","confidence":0.6,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"nist-ai-rmf/govern-1.5","regulationId":"nist-ai-rmf-1.0","article":"GOVERN 1.5","title":"Ongoing monitoring and periodic review of risk management","principle":"accountability","verdict":"partial","score":2,"rawScore":0.5,"evidence":[{"file":".github/workflows/ci.yaml","rule":"ci_eval_gates"}],"rules":[{"rule":"ci_eval_gates","matched":1,"weight":0.5},{"rule":"drift_monitoring_present","matched":0,"weight":0.5}],"rationale":"Composite raw score 0.50 (1/2 rules matched). Supporting docs may exist outside the repo.\n\nLLM judge (confidence 0.45): CI/CD evaluation gates demonstrate some monitoring capability, but the absence of documented drift monitoring and lack of evidence for defined organizational roles, responsibilities, and periodic review frequency prevents full compliance with GOVERN 1.5's comprehensive requirements.","confidence":0.45,"needsLlmJudge":true,"verified":true,"verifyMethod":"llm-judge"},{"clauseId":"nist-ai-rmf/map-1.1","regulationId":"nist-ai-rmf-1.0","article":"MAP 1.1","title":"Context of use established and understood","principle":"auditability","verdict":"fail","score":1,"rawScore":0.15,"evidence":[{"file":"README.md","rule":"readme_quality","detail":"1 required sections present"},{"file":"README.md","rule":"output_interpretation_guidance"}],"rules":[{"rule":"intended_use_documented","matched":0.25,"weight":0.6},{"rule":"deployment_context_documented","matched":0,"weight":0.4}],"rationale":"Composite raw score 0.15 (0/2 rules matched). Supporting docs may exist outside the repo.","confidence":0.75,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"nist-ai-rmf/map-3.4","regulationId":"nist-ai-rmf-1.0","article":"MAP 3.4","title":"Risks and benefits to people identified","principle":"safety","verdict":"fail","score":0,"rawScore":0,"evidence":[],"rules":[{"rule":"presence_of_risk_register","matched":0,"weight":0.6},{"rule":"presence_of_threat_model","matched":0,"weight":0.4}],"rationale":"Composite raw score 0.00 (0/2 rules matched). Supporting docs may exist outside the repo.","confidence":0.6,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"nist-ai-rmf/measure-2.3","regulationId":"nist-ai-rmf-1.0","article":"MEASURE 2.3","title":"AI system performance evaluated and documented","principle":"safety","verdict":"partial","score":2,"rawScore":0.7,"evidence":[{"file":"evals/","rule":"eval_suite_present"},{"file":"README.md","rule":"metrics_documented"},{"file":".github/workflows/ci.yaml","rule":"ci_eval_gates"}],"rules":[{"rule":"eval_suite_present","matched":1,"weight":0.5},{"rule":"metrics_documented","matched":0,"weight":0.3},{"rule":"eval_in_ci","matched":1,"weight":0.2}],"rationale":"Composite raw score 0.70 (2/3 rules matched).\n\nLLM judge (confidence 0.72): Evaluation suite and CI integration are present (0.7 raw score), but absence of documented metrics is a material gap for demonstrating measured performance criteria as required by the clause. Documentation of measures is explicitly mandated and currently missing.","confidence":0.72,"needsLlmJudge":true,"verified":true,"verifyMethod":"llm-judge"},{"clauseId":"nist-ai-rmf/measure-2.7","regulationId":"nist-ai-rmf-1.0","article":"MEASURE 2.7","title":"Security and resilience evaluated","principle":"security-governance","verdict":"fail","score":0,"rawScore":0.118169700377573,"evidence":[{"file":"gpt_engineer/core/ai.py","lines":[253,253],"snippet":"ackoff.expo, openai.RateLimitError, max_tries=7, max_time=45) def backoff_inference(self, messages): \"\"\" Perform inferen","rule":"rate_limit_usage"},{"file":"gpt_engineer/core/ai.py","lines":[278,278],"snippet":"openai.error.RateLimitError If the number of retries exceeds the maximum or if the rate limit persists beyond the","rule":"rate_limit_usage"},{"file":"gpt_engineer/core/ai.py","lines":[280,280],"snippet":"ultimately raise a RateLimitError. Example ------- >>> messages = [SystemMessage(content=\"Hello\"), HumanMessage(co","rule":"rate_limit_usage"}],"rules":[{"rule":"prompt_injection_defences","matched":0,"weight":0.4},{"rule":"adversarial_eval_present","matched":0,"weight":0.4},{"rule":"rate_limiting","matched":0.590848501887865,"weight":0.2}],"rationale":"Composite raw score 0.12 (1/3 rules matched).","confidence":0.768169700377573,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"nist-ai-rmf/measure-2.8","regulationId":"nist-ai-rmf-1.0","article":"MEASURE 2.8","title":"Privacy risk of the AI system evaluated","principle":"privacy","verdict":"fail","score":0,"rawScore":0,"evidence":[],"rules":[{"rule":"pii_redaction_present","matched":0,"weight":0.5},{"rule":"privacy_documentation","matched":0,"weight":0.5}],"rationale":"Composite raw score 0.00 (0/2 rules matched).","confidence":0.6,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"nist-ai-rmf/measure-2.11","regulationId":"nist-ai-rmf-1.0","article":"MEASURE 2.11","title":"Fairness and bias evaluated","principle":"fairness","verdict":"fail","score":0,"rawScore":0,"evidence":[],"rules":[{"rule":"bias_evaluation_present","matched":0,"weight":1}],"rationale":"Composite raw score 0.00 (0/1 rules matched).","confidence":0.55,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"nist-ai-rmf/manage-2.3","regulationId":"nist-ai-rmf-1.0","article":"MANAGE 2.3","title":"Mechanisms to supersede or deactivate AI systems","principle":"human-oversight","verdict":"fail","score":0,"rawScore":0,"evidence":[],"rules":[{"rule":"kill_switch_present","matched":0,"weight":0.6},{"rule":"feature_flag_for_disable","matched":0,"weight":0.4}],"rationale":"Composite raw score 0.00 (0/2 rules matched).","confidence":0.6,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"nist-ai-rmf/manage-4.1","regulationId":"nist-ai-rmf-1.0","article":"MANAGE 4.1","title":"Post-deployment monitoring, appeal and override, change management","principle":"auditability","verdict":"fail","score":1,"rawScore":0.36,"evidence":[{"file":"gpt_engineer/core/ai.py","lines":[33,33],"snippet":"AIMessage, HumanMessage, SystemMessage, messages_from_dict, messages_to_dict, ) from langchain_anthropic import ChatAnt","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[44,44],"snippet":"= Union[AIMessage, HumanMessage, SystemMessage] # Set up logging logger = logging.getLogger(__name__) class AI: \"\"\" A class that","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[141,141],"snippet":"ystem), HumanMessage(content=user), ] return self.next(messages, step_name=step_name) def _extract_content(","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[233,233],"snippet":"messages.append(HumanMessage(content=prompt)) logger.debug( \"Creating a new chat completion: %s\", \"\\n\".","rule":"human_in_loop"},{"file":"gpt_engineer/core/ai.py","lines":[284,284],"snippet":"e(content=\"Hello\"), HumanMessage(content=\"How's the weather?\")] >>> response = backoff_inference(messages) \"\"\" retur","rule":"human_in_loop"},{"file":"gpt_engineer/core/default/steps.py","lines":[42,42],"snippet":"chain.schema import HumanMessage, SystemMessage from termcolor import colored from gpt_engineer.core.ai import AI from gpt_engineer.core.ba","rule":"human_in_loop"}],"rules":[{"rule":"feedback_capture_present","matched":0,"weight":0.3},{"rule":"override_path_present","matched":1,"weight":0.3},{"rule":"structured_logging_imported","matched":0,"weight":0.2},{"rule":"versioning_visible","matched":0.3,"weight":0.2}],"rationale":"Composite raw score 0.36 (1/4 rules matched).","confidence":1,"needsLlmJudge":true,"verified":true,"verifyMethod":"skipped"},{"clauseId":"iso-42001/clause-5.1-leadership","regulationId":"iso-42001-2023","article":"5.1","title":"Leadership and commitment for AI management","principle":"accountability","verdict":"fail","score":0,"rawScore":0,"evidence":[],"rules":[{"rule":"presence_of_ai_policy","matched":0,"weight":0.7},{"rule":"leadership_signoff_evidence","matched":0,"weight":0.3}],"rationale":"Composite raw score 0.00 (0/2 rules matched). Supporting docs may exist outside the repo.","confidence":0.6,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"iso-42001/clause-5.3-roles","regulationId":"iso-42001-2023","article":"5.3","title":"Roles, responsibilities and authorities","principle":"accountability","verdict":"pass","score":4,"rawScore":1,"evidence":[{"file":".github/CODEOWNERS","rule":"presence_of_raci_or_owners"}],"rules":[{"rule":"presence_of_raci_or_owners","matched":1,"weight":1}],"rationale":"Composite raw score 1.00 (1/1 rules matched). Supporting docs may exist outside the repo.","confidence":0.55,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"iso-42001/clause-6.1-risk-assessment","regulationId":"iso-42001-2023","article":"6.1","title":"AI risk assessment process","principle":"safety","verdict":"fail","score":0,"rawScore":0,"evidence":[],"rules":[{"rule":"presence_of_risk_register","matched":0,"weight":0.6},{"rule":"risk_assessment_methodology_documented","matched":0,"weight":0.4}],"rationale":"Composite raw score 0.00 (0/2 rules matched). Supporting docs may exist outside the repo.","confidence":0.6,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"iso-42001/clause-7.5-documented-information","regulationId":"iso-42001-2023","article":"7.5","title":"Documented information for the AI management system","principle":"auditability","verdict":"pass","score":3,"rawScore":0.7,"evidence":[{"file":"README.md","rule":"presence_of_versioned_docs"},{"file":".github/workflows/release.yaml","rule":"docs_changelog_present"}],"rules":[{"rule":"presence_of_versioned_docs","matched":0.4,"weight":0.5},{"rule":"docs_changelog_present","matched":1,"weight":0.5}],"rationale":"Composite raw score 0.70 (1/2 rules matched). Supporting docs may exist outside the repo.","confidence":0.9,"needsLlmJudge":true,"verified":true,"verifyMethod":"skipped"},{"clauseId":"iso-42001/clause-8.1-operational-planning","regulationId":"iso-42001-2023","article":"8.1","title":"Operational planning and control","principle":"safety","verdict":"partial","score":2,"rawScore":0.5,"evidence":[{"file":".github/workflows/","rule":"presence_of_ci_workflows"}],"rules":[{"rule":"presence_of_runbook","matched":0,"weight":0.5},{"rule":"presence_of_ci_workflows","matched":1,"weight":0.5}],"rationale":"Composite raw score 0.50 (1/2 rules matched). Supporting docs may exist outside the repo.","confidence":1,"needsLlmJudge":true,"verified":true,"verifyMethod":"skipped"},{"clauseId":"iso-42001/clause-9.1-monitoring","regulationId":"iso-42001-2023","article":"9.1","title":"Monitoring, measurement, analysis and evaluation","principle":"auditability","verdict":"pass","score":3,"rawScore":0.75,"evidence":[{"file":"evals/","rule":"presence_of_eval_suite"},{"file":"gpt_engineer/applications/cli/main.py","lines":[30,30],"snippet":"difflib import json import logging import os import platform import subprocess import sys from pathlib import Path import openai import ty","rule":"structured_logging_present"},{"file":"gpt_engineer/core/ai.py","lines":[19,19],"snippet":"ations import json import logging import os from pathlib import Path from typing import Any, List, Optional, Union import backoff import","rule":"structured_logging_present"}],"rules":[{"rule":"presence_of_eval_suite","matched":1,"weight":0.5},{"rule":"structured_logging_present","matched":0.5,"weight":0.5}],"rationale":"Composite raw score 0.75 (2/2 rules matched). Supporting docs may exist outside the repo.","confidence":0.85,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"iso-42001/annex-a5-internal-org","regulationId":"iso-42001-2023","article":"A.5","title":"Internal organization controls","principle":"accountability","verdict":"pass","score":4,"rawScore":1,"evidence":[{"file":".github/CODEOWNERS","rule":"presence_of_codeowners"}],"rules":[{"rule":"presence_of_codeowners","matched":1,"weight":1}],"rationale":"Composite raw score 1.00 (1/1 rules matched). Supporting docs may exist outside the repo.","confidence":0.55,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"iso-42001/annex-a7-resources","regulationId":"iso-42001-2023","article":"A.7","title":"Resources for AI systems","principle":"security-governance","verdict":"partial","score":2,"rawScore":0.5,"evidence":[{"file":"poetry.lock","rule":"dependency_pinning"}],"rules":[{"rule":"presence_of_security_policy","matched":0,"weight":0.5},{"rule":"dependency_pinning","matched":1,"weight":0.5}],"rationale":"Composite raw score 0.50 (1/2 rules matched). Supporting docs may exist outside the repo.","confidence":1,"needsLlmJudge":true,"verified":true,"verifyMethod":"skipped"},{"clauseId":"gdpr/art-5-principles","regulationId":"gdpr-2016","article":"5","title":"Principles relating to processing of personal data","principle":"transparency","verdict":"fail","score":0,"rawScore":0,"evidence":[],"rules":[{"rule":"presence_of_privacy_policy","matched":0,"weight":0.5},{"rule":"purpose_limitation_documented","matched":0,"weight":0.5}],"rationale":"Composite raw score 0.00 (0/2 rules matched). Supporting docs may exist outside the repo.","confidence":0.6,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"gdpr/art-22-automated-decisions","regulationId":"gdpr-2016","article":"22","title":"Automated individual decision-making, including profiling","principle":"human-oversight","verdict":"pass","score":4,"rawScore":1,"evidence":[{"file":"gpt_engineer/core/ai.py","lines":[33,33],"snippet":"AIMessage, HumanMessage, SystemMessage, messages_from_dict, messages_to_dict, ) from langchain_anthropic import ChatAnt","rule":"human_review_path_present"},{"file":"gpt_engineer/core/ai.py","lines":[44,44],"snippet":"= Union[AIMessage, HumanMessage, SystemMessage] # Set up logging logger = logging.getLogger(__name__) class AI: \"\"\" A class that","rule":"human_review_path_present"},{"file":"gpt_engineer/core/ai.py","lines":[141,141],"snippet":"ystem), HumanMessage(content=user), ] return self.next(messages, step_name=step_name) def _extract_content(","rule":"human_review_path_present"},{"file":"gpt_engineer/core/ai.py","lines":[233,233],"snippet":"messages.append(HumanMessage(content=prompt)) logger.debug( \"Creating a new chat completion: %s\", \"\\n\".","rule":"human_review_path_present"},{"file":"gpt_engineer/core/ai.py","lines":[284,284],"snippet":"e(content=\"Hello\"), HumanMessage(content=\"How's the weather?\")] >>> response = backoff_inference(messages) \"\"\" retur","rule":"human_review_path_present"},{"file":"gpt_engineer/core/default/steps.py","lines":[42,42],"snippet":"chain.schema import HumanMessage, SystemMessage from termcolor import colored from gpt_engineer.core.ai import AI from gpt_engineer.core.ba","rule":"human_review_path_present"},{"file":"gpt_engineer/core/ai.py","lines":[33,33],"snippet":"AIMessage, HumanMessage, SystemMessage, messages_from_dict, messages_to_dict, ) from langchain_anthropic import ChatAnt","rule":"appeal_or_override_mechanism"},{"file":"gpt_engineer/core/ai.py","lines":[44,44],"snippet":"= Union[AIMessage, HumanMessage, SystemMessage] # Set up logging logger = logging.getLogger(__name__) class AI: \"\"\" A class that","rule":"appeal_or_override_mechanism"},{"file":"gpt_engineer/core/ai.py","lines":[141,141],"snippet":"ystem), HumanMessage(content=user), ] return self.next(messages, step_name=step_name) def _extract_content(","rule":"appeal_or_override_mechanism"},{"file":"gpt_engineer/core/ai.py","lines":[233,233],"snippet":"messages.append(HumanMessage(content=prompt)) logger.debug( \"Creating a new chat completion: %s\", \"\\n\".","rule":"appeal_or_override_mechanism"},{"file":"gpt_engineer/core/ai.py","lines":[284,284],"snippet":"e(content=\"Hello\"), HumanMessage(content=\"How's the weather?\")] >>> response = backoff_inference(messages) \"\"\" retur","rule":"appeal_or_override_mechanism"},{"file":"gpt_engineer/core/default/steps.py","lines":[42,42],"snippet":"chain.schema import HumanMessage, SystemMessage from termcolor import colored from gpt_engineer.core.ai import AI from gpt_engineer.core.ba","rule":"appeal_or_override_mechanism"}],"rules":[{"rule":"human_review_path_present","matched":1,"weight":0.6},{"rule":"appeal_or_override_mechanism","matched":1,"weight":0.4}],"rationale":"Composite raw score 1.00 (2/2 rules matched). Supporting docs may exist outside the repo.","confidence":0.6,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"gdpr/art-25-by-design","regulationId":"gdpr-2016","article":"25","title":"Data protection by design and by default","principle":"privacy","verdict":"pass","score":4,"rawScore":1,"evidence":[{"file":"gpt_engineer/applications/cli/collect.py","lines":[138,138],"snippet":"hashlib.sha256","rule":"pseudonymisation_or_anonymisation"},{"file":"README.md","rule":"default_minimal_data_collection"}],"rules":[{"rule":"pseudonymisation_or_anonymisation","matched":1,"weight":0.5},{"rule":"default_minimal_data_collection","matched":1,"weight":0.5}],"rationale":"Composite raw score 1.00 (2/2 rules matched). Supporting docs may exist outside the repo.","confidence":0.6,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"gdpr/art-30-records","regulationId":"gdpr-2016","article":"30","title":"Records of processing activities","principle":"auditability","verdict":"fail","score":0,"rawScore":0,"evidence":[],"rules":[{"rule":"presence_of_processing_register","matched":0,"weight":1}],"rationale":"Composite raw score 0.00 (0/1 rules matched). Supporting docs may exist outside the repo.","confidence":0.55,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"},{"clauseId":"gdpr/art-32-security","regulationId":"gdpr-2016","article":"32","title":"Security of processing","principle":"security-governance","verdict":"partial","score":2,"rawScore":0.5,"evidence":[{"file":"gpt_engineer/applications/cli/collect.py","lines":[56,56],"snippet":"https://gptengineerezm.dataplane.rudderstack.com","rule":"encryption_at_rest_or_transit"},{"file":"gpt_engineer/applications/cli/main.py","lines":[320,320],"snippet":"https://xx.openai.azure.com).","rule":"encryption_at_rest_or_transit"},{"file":"gpt_engineer/core/project_config.py","lines":[28,28],"snippet":"https://api.gptengineer.app/openapi.json","rule":"encryption_at_rest_or_transit"},{"file":"gpt_engineer/core/token_usage.py","lines":[15,15],"snippet":"https://github.com/langchain-ai/langchain/blob/535db72607c4ae308566ede4af65295967bb33a8/libs/community/langchain_community/callbacks/openai_info.py","rule":"encryption_at_rest_or_transit"}],"rules":[{"rule":"encryption_at_rest_or_transit","matched":1,"weight":0.5},{"rule":"access_control_enforcement","matched":0,"weight":0.5}],"rationale":"Composite raw score 0.50 (1/2 rules matched). Supporting docs may exist outside the repo.","confidence":1,"needsLlmJudge":true,"verified":true,"verifyMethod":"skipped"},{"clauseId":"gdpr/art-35-dpia","regulationId":"gdpr-2016","article":"35","title":"Data protection impact assessment (DPIA)","principle":"safety","verdict":"fail","score":0,"rawScore":0,"evidence":[],"rules":[{"rule":"presence_of_dpia","matched":0,"weight":1}],"rationale":"Composite raw score 0.00 (0/1 rules matched). Supporting docs may exist outside the repo.","confidence":0.55,"needsLlmJudge":false,"verified":true,"verifyMethod":"deterministic-only"}],"grade":{"overallScore":1.6666666666666667,"perRegulation":[{"regulationId":"eu-ai-act-2024-08","scoreAvg":1.9523809523809523,"passCount":7,"partialCount":6,"failCount":8,"externalCount":1,"naCount":3},{"regulationId":"nist-ai-rmf-1.0","scoreAvg":0.6,"passCount":0,"partialCount":2,"failCount":8,"externalCount":0,"naCount":0},{"regulationId":"iso-42001-2023","scoreAvg":2.25,"passCount":4,"partialCount":2,"failCount":2,"externalCount":0,"naCount":0},{"regulationId":"gdpr-2016","scoreAvg":1.6666666666666667,"passCount":2,"partialCount":1,"failCount":3,"externalCount":0,"naCount":0}]},"bundleHash":"28ffdb8667786fb772e3cb282cb5e7315194c72b6eefafec1725710d921d3439","checkerVersion":"v0.1.0","regulationsVersions":{"eu-ai-act-2024-08":"ee8568c60676d1c60c00393077dc7b9753628a16bb96049d6caa384b93516239","nist-ai-rmf-1.0":"b36616651114a74e212e4416e72f3d320d3ea46e1c93d7d3919cd44b5f71f762","iso-42001-2023":"3855f297eea17980b71d32e9d8c9f113c153c68e07b01fd442392fba4567755d","gdpr-2016":"a650813378c93140abcbda864a7248289096abec98d57667cb0c0e2cba1bfe3f"},"startedAt":"2026-05-22T04:53:06.537Z","completedAt":"2026-05-22T04:53:39.928Z","durationMs":33391,"externalControls":[{"clauseId":"eu-ai-act/art-5/1h-realtime-rbi","article":"5(1)(h)","title":"Real-time remote biometric identification in public spaces","note":"Deployment context (public space, real-time, law enforcement use, judicial\nauthorisation) is operational, not knowable from code. Always external."}]}}