{ "what": "MIG partitioning state of the reference H200: what the device can be carved into, what is currently carved, and what holds it. No benchmark was run and none is reported.", "schema_version": "mig-observe/1", "host": "h200-ref", "gpu": "NVIDIA H200 NVL, 143771 MiB total", "date_utc": "2026-08-04", "measured_at": "2026-08-04T12:20:25+00:00", "producer": "scripts/cotenancy/mig-h200-probe.py", "producer_mode": "observe", "reproduce": "WG3_MIG_TARGET= python3 scripts/cotenancy/mig-h200-probe.py --mode observe", "target": { "kind": "ssh", "resolved_from": "--host / $WG3_MIG_TARGET", "value_recorded": false, "why": "an ssh destination is an internal host name or a tailnet address; tests/publication rejects both in a tracked file." }, "posture": { "read_only": true, "why": "the device is shared and in production use. This mode issues nvidia-smi queries only; it holds no CUDA context, changes no device state and touches no process." }, "measurement_states": { "_what": "the three-state envelope every value below carries. Mirrors services/ran-manager/main.py; `simulated` is deliberately not offered here and `absent` is added.", "measured": "we asked and this is the answer", "absent": "we asked and the entity does not exist right now; value is null", "unknown": "we could not ask or could not establish it; value is null" }, "observed": { "machine": { "value": { "arch_kernel": "5.14.0-687.17.1.el9_8.x86_64 x86_64", "reference_label": "h200-ref" }, "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "uname -m -r", "exit_code": 0, "stdout": "5.14.0-687.17.1.el9_8.x86_64 x86_64", "stderr": "" }, "unknown_reason": null }, "gpu_name": { "value": "NVIDIA H200 NVL", "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi --query-gpu=index,name,uuid,memory.total,memory.used,mig.mode.current,mig.mode.pending,driver_version --format=csv,noheader,nounits", "exit_code": 0, "stdout": "0, NVIDIA H200 NVL, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 143771, 141643, Disabled, Disabled, 590.48.01", "stderr": "" }, "unknown_reason": null }, "gpu_uuid": { "value": "GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c", "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi --query-gpu=index,name,uuid,memory.total,memory.used,mig.mode.current,mig.mode.pending,driver_version --format=csv,noheader,nounits", "exit_code": 0, "stdout": "0, NVIDIA H200 NVL, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 143771, 141643, Disabled, Disabled, 590.48.01", "stderr": "" }, "unknown_reason": null }, "driver_version": { "value": "590.48.01", "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi --query-gpu=index,name,uuid,memory.total,memory.used,mig.mode.current,mig.mode.pending,driver_version --format=csv,noheader,nounits", "exit_code": 0, "stdout": "0, NVIDIA H200 NVL, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 143771, 141643, Disabled, Disabled, 590.48.01", "stderr": "" }, "unknown_reason": null }, "memory_total_mib": { "value": 143771, "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi --query-gpu=index,name,uuid,memory.total,memory.used,mig.mode.current,mig.mode.pending,driver_version --format=csv,noheader,nounits", "exit_code": 0, "stdout": "0, NVIDIA H200 NVL, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 143771, 141643, Disabled, Disabled, 590.48.01", "stderr": "" }, "unknown_reason": null, "unit": "MiB" }, "memory_used_mib": { "value": 141643, "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi --query-gpu=index,name,uuid,memory.total,memory.used,mig.mode.current,mig.mode.pending,driver_version --format=csv,noheader,nounits", "exit_code": 0, "stdout": "0, NVIDIA H200 NVL, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 143771, 141643, Disabled, Disabled, 590.48.01", "stderr": "" }, "unknown_reason": null, "unit": "MiB" }, "mig_mode_current": { "value": "Disabled", "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi --query-gpu=index,name,uuid,memory.total,memory.used,mig.mode.current,mig.mode.pending,driver_version --format=csv,noheader,nounits", "exit_code": 0, "stdout": "0, NVIDIA H200 NVL, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 143771, 141643, Disabled, Disabled, 590.48.01", "stderr": "" }, "unknown_reason": null }, "mig_mode_pending": { "value": "Disabled", "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi --query-gpu=index,name,uuid,memory.total,memory.used,mig.mode.current,mig.mode.pending,driver_version --format=csv,noheader,nounits", "exit_code": 0, "stdout": "0, NVIDIA H200 NVL, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 143771, 141643, Disabled, Disabled, 590.48.01", "stderr": "" }, "unknown_reason": null }, "profiles_available": { "value": [ { "gpu": 0, "name": "1g.18gb", "profile_id": 19, "instances_free": 7, "instances_total": 7, "memory_gib": 16.0, "p2p": "No", "sm": 16, "dec": 1, "enc": 0, "ce": 1, "jpeg": 1, "ofa": 0 }, { "gpu": 0, "name": "1g.18gb+me", "profile_id": 20, "instances_free": 1, "instances_total": 1, "memory_gib": 16.0, "p2p": "No", "sm": 16, "dec": 1, "enc": 0, "ce": 1, "jpeg": 1, "ofa": 1 }, { "gpu": 0, "name": "1g.35gb", "profile_id": 15, "instances_free": 4, "instances_total": 4, "memory_gib": 32.5, "p2p": "No", "sm": 26, "dec": 1, "enc": 0, "ce": 1, "jpeg": 1, "ofa": 0 }, { "gpu": 0, "name": "2g.35gb", "profile_id": 14, "instances_free": 3, "instances_total": 3, "memory_gib": 32.5, "p2p": "No", "sm": 32, "dec": 2, "enc": 0, "ce": 2, "jpeg": 2, "ofa": 0 }, { "gpu": 0, "name": "3g.71gb", "profile_id": 9, "instances_free": 2, "instances_total": 2, "memory_gib": 69.75, "p2p": "No", "sm": 60, "dec": 3, "enc": 0, "ce": 3, "jpeg": 3, "ofa": 0 }, { "gpu": 0, "name": "4g.71gb", "profile_id": 5, "instances_free": 1, "instances_total": 1, "memory_gib": 69.75, "p2p": "No", "sm": 64, "dec": 4, "enc": 0, "ce": 4, "jpeg": 4, "ofa": 0 }, { "gpu": 0, "name": "7g.141gb", "profile_id": 0, "instances_free": 1, "instances_total": 1, "memory_gib": 140.0, "p2p": "No", "sm": 132, "dec": 7, "enc": 0, "ce": 8, "jpeg": 7, "ofa": 1 } ], "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi mig -lgip", "exit_code": 0, "stdout": "+-------------------------------------------------------------------------------+\n| GPU instance profiles: |\n| GPU Name ID Instances Memory P2P SM DEC ENC |\n| Free/Total GiB CE JPEG OFA |\n|===============================================================================|\n| 0 MIG 1g.18gb 19 7/7 16.00 No 16 1 0 |\n| 1 1 0 |\n+-------------------------------------------------------------------------------+\n| 0 MIG 1g.18gb+me 20 1/1 16.00 No 16 1 0 |\n| 1 1 1 |\n+-------------------------------------------------------------------------------+\n| 0 MIG 1g.35gb 15 4/4 32.50 No 26 1 0 |\n| 1 1 0 |\n+-------------------------------------------------------------------------------+\n| 0 MIG 2g.35gb 14 3/3 32.50 No 32 2 0 |\n| 2 2 0 |\n+-------------------------------------------------------------------------------+\n| 0 MIG 3g.71gb 9 2/2 69.75 No 60 3 0 |\n| 3 3 0 |\n+-------------------------------------------------------------------------------+\n| 0 MIG 4g.71gb 5 1/1 69.75 No 64 4 0 |\n| 4 4 0 |\n+-------------------------------------------------------------------------------+\n| 0 MIG 7g.141gb 0 1/1 140.00 No 132 7 0 |\n| 8 7 1 |\n+-------------------------------------------------------------------------------+", "stderr": "" }, "unknown_reason": null, "note": "readable whether or not MIG is enabled: this is the device's partitioning CAPABILITY, not its current partitioning." }, "profiles_available_free_by_name": { "value": { "1g.18gb": 7, "1g.18gb+me": 1, "1g.35gb": 4, "2g.35gb": 3, "3g.71gb": 2, "4g.71gb": 1, "7g.141gb": 1 }, "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "derived_from": "profiles_available" }, "unknown_reason": null, "note": "the shape the 2026-08-03 artefact recorded, for comparison." }, "gpu_instances": { "value": null, "state": "absent", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi mig -lgi", "exit_code": 0, "stdout": "No MIG-enabled devices found.", "stderr": "" }, "unknown_reason": "`nvidia-smi mig -lgi` answered, and the answer is that no GPU instances exist: MIG mode is not enabled on this device, so there is nothing to list. This is an absence, not a count of zero." }, "compute_instances": { "value": null, "state": "absent", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi mig -lci", "exit_code": 0, "stdout": "No MIG-enabled devices found.", "stderr": "" }, "unknown_reason": "`nvidia-smi mig -lci` answered, and the answer is that no compute instances exist: MIG mode is not enabled on this device, so there is nothing to list. This is an absence, not a count of zero." }, "compute_apps": { "value": [ { "pid": 1793709, "process_name": "VLLM::EngineCore", "gpu_uuid": "GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c", "used_memory_mib": 95772 }, { "pid": 4026373, "process_name": "/usr/local/lib/ollama/llama-server", "gpu_uuid": "GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c", "used_memory_mib": 45856 } ], "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi --query-compute-apps=pid,process_name,gpu_uuid,used_gpu_memory --format=csv,noheader,nounits", "exit_code": 0, "stdout": "1793709, VLLM::EngineCore, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 95772\n4026373, /usr/local/lib/ollama/llama-server, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 45856", "stderr": "" }, "unknown_reason": null } }, "reshape_preconditions": { "_what": "what would have to happen before MIG on this GPU could be reshaped. Computed from this probe round, not carried from a previous one.", "rule": "a MIG mode change requires a compute-free GPU: the driver refuses while any CUDA context exists on the device.", "gpu_is_compute_free": { "value": false, "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "process_count": 2 }, "unknown_reason": null }, "processes_holding_the_gpu": { "value": [ { "pid": 1793709, "process_name": "VLLM::EngineCore", "gpu_uuid": "GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c", "used_memory_mib": 95772 }, { "pid": 4026373, "process_name": "/usr/local/lib/ollama/llama-server", "gpu_uuid": "GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c", "used_memory_mib": 45856 } ], "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi --query-compute-apps=pid,process_name,gpu_uuid,used_gpu_memory --format=csv,noheader,nounits", "exit_code": 0, "stdout": "1793709, VLLM::EngineCore, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 95772\n4026373, /usr/local/lib/ollama/llama-server, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 45856", "stderr": "" }, "unknown_reason": null, "note": "every CUDA process currently holding the device. A MIG mode change requires this list to be EMPTY." }, "memory_held_by_those_processes_mib": { "value": 141628, "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi --query-compute-apps=pid,process_name,gpu_uuid,used_gpu_memory --format=csv,noheader,nounits", "exit_code": 0, "stdout": "1793709, VLLM::EngineCore, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 95772\n4026373, /usr/local/lib/ollama/llama-server, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 45856", "stderr": "" }, "unknown_reason": null, "unit": "MiB", "note": "memory that would have to be given up by other tenants." }, "root_privilege_available_non_interactively": { "value": true, "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "sudo -n true && echo PASSWORDLESS_SUDO=yes || echo PASSWORDLESS_SUDO=no", "exit_code": 0, "stdout": "PASSWORDLESS_SUDO=yes", "stderr": "" }, "unknown_reason": null, "note": "`nvidia-smi -i -mig <0|1>` is a privileged operation. false means an interactive operator is required." }, "gpu_reset_or_driver_reload_required": { "value": null, "state": "unknown", "source": "nvidia-smi on the target", "measured_at": null, "evidence": { "command": "nvidia-smi -q -i 0 | grep -m1 -A2 'MIG Mode'", "exit_code": 0, "stdout": "MIG Mode\n Current : Disabled\n Pending : Disabled", "stderr": "" }, "unknown_reason": "MIG mode current and pending are both 'Disabled', so there is no pending change whose application could be observed. Whether a reset would be needed AFTER a future `nvidia-smi -mig 1` is not determinable from this state: on some driver and platform combinations the change applies immediately on a compute-free GPU, and on others nvidia-smi reports a pending-enable state and asks for a reset. It becomes observable the moment the change is attempted (pending will then differ from current). This producer will not guess." }, "reset_context": { "value": { "persistence_mode": "Disabled", "ecc_mode_current": "Enabled", "ecc_mode_pending": "Enabled" }, "state": "measured", "source": "nvidia-smi on the target", "measured_at": "2026-08-04T12:20:25+00:00", "evidence": { "command": "nvidia-smi --query-gpu=persistence_mode,ecc.mode.current,ecc.mode.pending --format=csv,noheader,nounits", "exit_code": 0, "stdout": "Disabled, Enabled, Enabled", "stderr": "" }, "unknown_reason": null }, "this_producer_will_not": [ "stop, kill or signal any process on the target", "enable or disable MIG", "create or destroy a GPU or compute instance" ] }, "prior_window": { "_what": "the prior operator window, carried forward so its unresolved finding survives the arrival of newer artefacts.", "instance_profile": { "value": "1g.18gb", "state": "measured", "source": "operator session recorded in benchmarks/reports/mig-h200-2026-08-03.json", "measured_at": null, "evidence": { "date_utc": "2026-08-03", "precision": "calendar day, not a measurement instant" }, "unknown_reason": null, "carried_forward": true }, "instances_created": { "value": 3, "state": "measured", "source": "operator session recorded in benchmarks/reports/mig-h200-2026-08-03.json", "measured_at": null, "evidence": { "date_utc": "2026-08-03", "precision": "calendar day, not a measurement instant" }, "unknown_reason": null, "carried_forward": true }, "memory_visible_to_process_gb": { "value": 17.2, "state": "measured", "source": "operator session recorded in benchmarks/reports/mig-h200-2026-08-03.json", "measured_at": null, "evidence": { "date_utc": "2026-08-03", "precision": "calendar day, not a measurement instant", "raw_evidence_on_target": { "harness": "/tmp/mig_probe.py", "surviving_output": "/tmp/mig_c1.json", "content": { "visible": "MIG-c06e7bf0-…", "total_mem_gb": 17.2, "tflops": 93.6 }, "status": "left in place, not removed — it is the only surviving raw evidence for the concurrent instance-1 figure and for the 17.2 GB visibility claim. It is scratch on a machine this project is a guest on and should be copied into the repository by its owner, not deleted by a reader." } }, "unknown_reason": null, "carried_forward": true, "note": "17.2 of 141 visible to the process — a hard partition, which MPS cannot do." }, "fp16_4096_matmul_tflops_per_instance_sequential": { "value": [ 93.64, 93.38, 93.29 ], "state": "measured", "source": "operator session recorded in benchmarks/reports/mig-h200-2026-08-03.json", "measured_at": null, "evidence": { "date_utc": "2026-08-03", "precision": "calendar day, not a measurement instant" }, "unknown_reason": null, "carried_forward": true, "note": "one instance at a time; not a concurrency result." }, "fp16_4096_matmul_tflops_instance_1_concurrent": { "value": 93.6, "state": "measured", "source": "operator session recorded in benchmarks/reports/mig-h200-2026-08-03.json", "measured_at": null, "evidence": { "date_utc": "2026-08-03", "precision": "calendar day, not a measurement instant", "raw_evidence_on_target": { "harness": "/tmp/mig_probe.py", "surviving_output": "/tmp/mig_c1.json", "content": { "visible": "MIG-c06e7bf0-…", "total_mem_gb": 17.2, "tflops": 93.6 }, "status": "left in place, not removed — it is the only surviving raw evidence for the concurrent instance-1 figure and for the 17.2 GB visibility claim. It is scratch on a machine this project is a guest on and should be copied into the repository by its owner, not deleted by a reader." } }, "unknown_reason": null, "carried_forward": true, "note": "instance 1 only. The other two are the unknown below." }, "three_way_concurrent_isolation": { "value": null, "state": "unknown", "source": "operator session recorded in benchmarks/reports/mig-h200-2026-08-03.json", "measured_at": null, "evidence": null, "unknown_reason": "STILL UNRESOLVED. In the 2026-08-03 window instances 2 and 3 produced NO output when all three ran at once, although each worked when run alone. The cause was never diagnosed — the window had to close to restore a user-facing service — so the concurrent non-interference claim is UNPROVEN and is not made. It is not zero, it is not a failure of isolation, and it is not a pass: it is a measurement that did not happen. NEW EVIDENCE, 2026-08-04: the operator's scratch survives on the target, and /tmp/mig_c1.json exists while mig_c2.json and mig_c3.json were never created at all. So instances 2 and 3 did not produce a WRONG number and did not produce a SLOW one — their processes never reached the print. That narrows the cause to launch or CUDA-context creation rather than to interference between instances, and it is consistent with the harness (/tmp/mig_probe.py) capturing no stderr, so whatever the two processes said on the way out was discarded. This is a lead, NOT a resolution: nothing here establishes what happened, and the isolation claim stays unmade. Resolving it needs a fresh compute-free window; `--mode measure` captures each instance's exit code AND stderr to its own file so a repeat cannot be silent a second time.", "carried_forward": true }, "cuphy_ldpc_inside_a_mig_instance": { "value": null, "state": "unknown", "source": "operator session recorded in benchmarks/reports/mig-h200-2026-08-03.json", "measured_at": null, "evidence": null, "unknown_reason": "not attempted in the 2026-08-03 window, and not attempted since.", "carried_forward": true } }, "not_measured_in_this_mode": { "_what": "observe mode reports no benchmark. These are the values the prior artefact carries and this run did NOT reproduce, listed so their absence is explicit rather than inferred from a missing key.", "fp16_4096_matmul_tflops_per_instance": { "value": null, "state": "unknown", "source": "scripts/cotenancy/mig-h200-probe.py", "measured_at": null, "evidence": null, "unknown_reason": "observe mode runs no CUDA work at all. Reproducing it requires --mode measure, which requires a compute-free GPU and an operator." }, "memory_visible_to_process_inside_an_instance_gb": { "value": null, "state": "unknown", "source": "scripts/cotenancy/mig-h200-probe.py", "measured_at": null, "evidence": null, "unknown_reason": "no instance exists to run inside; MIG is disabled on this device." }, "three_way_concurrent_isolation": { "value": null, "state": "unknown", "source": "scripts/cotenancy/mig-h200-probe.py", "measured_at": null, "evidence": null, "unknown_reason": "carried forward unresolved from the 2026-08-03 window; see prior_window.three_way_concurrent_isolation. This run did not attempt it and does not resolve it." } }, "evidence_raw": { "compute_apps": { "command": "nvidia-smi --query-compute-apps=pid,process_name,gpu_uuid,used_gpu_memory --format=csv,noheader,nounits", "exit_code": 0, "stdout": "1793709, VLLM::EngineCore, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 95772\n4026373, /usr/local/lib/ollama/llama-server, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 45856", "stderr": "" }, "compute_instances": { "command": "nvidia-smi mig -lci", "exit_code": 0, "stdout": "No MIG-enabled devices found.", "stderr": "" }, "gip_profiles": { "command": "nvidia-smi mig -lgip", "exit_code": 0, "stdout": "+-------------------------------------------------------------------------------+\n| GPU instance profiles: |\n| GPU Name ID Instances Memory P2P SM DEC ENC |\n| Free/Total GiB CE JPEG OFA |\n|===============================================================================|\n| 0 MIG 1g.18gb 19 7/7 16.00 No 16 1 0 |\n| 1 1 0 |\n+-------------------------------------------------------------------------------+\n| 0 MIG 1g.18gb+me 20 1/1 16.00 No 16 1 0 |\n| 1 1 1 |\n+-------------------------------------------------------------------------------+\n| 0 MIG 1g.35gb 15 4/4 32.50 No 26 1 0 |\n| 1 1 0 |\n+-------------------------------------------------------------------------------+\n| 0 MIG 2g.35gb 14 3/3 32.50 No 32 2 0 |\n| 2 2 0 |\n+-------------------------------------------------------------------------------+\n| 0 MIG 3g.71gb 9 2/2 69.75 No 60 3 0 |\n| 3 3 0 |\n+-------------------------------------------------------------------------------+\n| 0 MIG 4g.71gb 5 1/1 69.75 No 64 4 0 |\n| 4 4 0 |\n+-------------------------------------------------------------------------------+\n| 0 MIG 7g.141gb 0 1/1 140.00 No 132 7 0 |\n| 8 7 1 |\n+-------------------------------------------------------------------------------+", "stderr": "" }, "gpu_instances": { "command": "nvidia-smi mig -lgi", "exit_code": 0, "stdout": "No MIG-enabled devices found.", "stderr": "" }, "gpu_query": { "command": "nvidia-smi --query-gpu=index,name,uuid,memory.total,memory.used,mig.mode.current,mig.mode.pending,driver_version --format=csv,noheader,nounits", "exit_code": 0, "stdout": "0, NVIDIA H200 NVL, GPU-1f3c24ab-965f-5b9a-5aca-a73200df296c, 143771, 141643, Disabled, Disabled, 590.48.01", "stderr": "" }, "gpu_query_reset_relevant": { "command": "nvidia-smi --query-gpu=persistence_mode,ecc.mode.current,ecc.mode.pending --format=csv,noheader,nounits", "exit_code": 0, "stdout": "Disabled, Enabled, Enabled", "stderr": "" }, "hostname": { "command": "hostname", "exit_code": 0, "stdout": "h200-ref", "stderr": "" }, "mig_mode_block": { "command": "nvidia-smi -q -i 0 | grep -m1 -A2 'MIG Mode'", "exit_code": 0, "stdout": "MIG Mode\n Current : Disabled\n Pending : Disabled", "stderr": "" }, "nvidia_smi_version": { "command": "nvidia-smi --version", "exit_code": 0, "stdout": "NVIDIA-SMI version : 590.48.01\nNVML version : 590.48\nDRIVER version : 590.48.01\nCUDA Version : 13.1", "stderr": "" }, "privilege": { "command": "sudo -n true && echo PASSWORDLESS_SUDO=yes || echo PASSWORDLESS_SUDO=no", "exit_code": 0, "stdout": "PASSWORDLESS_SUDO=yes", "stderr": "" }, "uname": { "command": "uname -m -r", "exit_code": 0, "stdout": "5.14.0-687.17.1.el9_8.x86_64 x86_64", "stderr": "" } }, "verify_against": { "prior_artefact": "benchmarks/reports/mig-h200-2026-08-03.json", "_what": "structural drift of this observation against benchmarks/reports/mig-h200-2026-08-03.json-shaped prior artefact. Numeric benchmark results are NOT compared; see verify_against() for why.", "compared_at": "2026-08-04T12:20:25+00:00", "drift_count": 3, "checks": [ { "claim": "gpu", "prior": "NVIDIA H200 NVL, 141 GB", "now": "NVIDIA H200 NVL, 143771 MiB total", "verdict": "drift", "note": "the prior file wrote memory in GB prose; this one writes the MiB the driver reports. A difference here is usually units, not hardware." }, { "claim": "host", "prior": "h200-ref", "now": "h200-ref", "verdict": "same" }, { "claim": "instance_profile", "prior": "1g.18gb", "now": "1g.18gb", "verdict": "same" }, { "claim": "instances_created", "prior": 3, "now": 3, "verdict": "same" }, { "claim": "profiles_available", "prior": { "1g.18gb": 7, "1g.18gb+me": 1, "3g.71gb": 2, "7g.141gb": 1 }, "now": { "1g.18gb": 7, "1g.18gb+me": 1, "1g.35gb": 4, "2g.35gb": 3, "3g.71gb": 2, "4g.71gb": 1, "7g.141gb": 1 }, "verdict": "drift" }, { "claim": "profiles_available.detail", "in_prior_not_now": [], "in_now_not_prior": [ "1g.35gb", "2g.35gb", "4g.71gb" ], "free_count_changed": {}, "verdict": "drift", "note": "a profile present now but absent from the prior record is usually an INCOMPLETE prior record rather than a hardware change: `mig -lgip` lists every profile the device supports and does not change between windows." }, { "claim": "three_way_concurrent_isolation", "prior": "recorded under NOT_measured", "now": "unknown", "verdict": "carried_forward_unresolved", "note": "this must never silently become measured. If it ever reads `measured`, a window resolved it and the evidence must say which." } ] } }