)'"
+ )
+ symbol_number = int(m_sym.group("number"))
+ if symbol_number != raw_sg_number:
+ raise ValueError(
+ f"phenix_payload.crystal_symmetry.space_group_symbol {space_group_symbol!r} "
+ f"declares space-group number {symbol_number}, which disagrees with "
+ f"space_group_number {raw_sg_number}"
+ )
+ normalized_lookup_symbol = "".join(m_sym.group("lookup_symbol").upper().split())
+ expected_input_symbol = input_symmetry.get("space_group_symbol")
+ if type(expected_input_symbol) is not str or not expected_input_symbol.strip():
+ raise ValueError(
+ "input_ledger.crystallographic_context.symmetry.space_group_symbol must be a non-empty string"
+ )
+ normalized_expected_symbol = "".join(expected_input_symbol.upper().split())
+ if normalized_lookup_symbol != normalized_expected_symbol:
+ raise ValueError(
+ f"phenix_payload.crystal_symmetry.space_group_symbol {space_group_symbol!r} "
+ f"(normalized symbol {normalized_lookup_symbol!r}) disagrees with "
+ f"input_ledger space_group_symbol {normalized_expected_symbol!r}"
+ )
+ expected_lattice = input_symmetry.get("lattice")
+ if (
+ type(expected_lattice) is not str
+ or not expected_lattice.strip()
+ or not normalized_lookup_symbol.startswith(expected_lattice.strip().upper())
+ ):
+ raise ValueError(
+ f"phenix_payload.crystal_symmetry.space_group_symbol {space_group_symbol!r} "
+ f"lattice disagrees with input_ledger lattice {expected_lattice!r}"
+ )
+
+ if maps_symmetry is not None:
+ if not isinstance(maps_symmetry, Mapping):
+ raise ValueError("map_mtz.crystallographic_context missing symmetry mapping")
+ maps_sg_number = maps_symmetry.get("space_group_number")
+ if maps_sg_number != raw_sg_number:
+ raise ValueError(
+ f"phenix_payload.crystal_symmetry.space_group_number {raw_sg_number} "
+ f"disagrees with map MTZ space_group_number {maps_sg_number!r}"
+ )
+ maps_ops_raw = maps_symmetry.get("operators_xyz")
+ if not isinstance(maps_ops_raw, (list, tuple)):
+ raise ValueError("map_mtz.crystallographic_context.symmetry missing operators_xyz")
+ normalized_maps_ops = sorted(
+ normalize_symmetry_operator_xyz(
+ op,
+ label=f"map_mtz.crystallographic_context.symmetry.operators_xyz[{idx}]",
+ )
+ for idx, op in enumerate(maps_ops_raw)
+ )
+ if sorted_runtime_ops != normalized_maps_ops:
+ raise ValueError(
+ "phenix_payload.crystal_symmetry.space_group_operators_xyz "
+ f"(normalized={sorted_runtime_ops!r}) disagrees with map MTZ "
+ f"symmetry.operators_xyz {normalized_maps_ops!r}"
+ )
+ maps_symbol = maps_symmetry.get("space_group_symbol")
+ if (
+ type(maps_symbol) is not str
+ or "".join(maps_symbol.upper().split()) != normalized_lookup_symbol
+ ):
+ raise ValueError(
+ f"phenix_payload.crystal_symmetry.space_group_symbol {space_group_symbol!r} "
+ f"disagrees with map MTZ space_group_symbol {maps_symbol!r}"
+ )
+
+ return {
+ "space_group_symbol": space_group_symbol,
+ "space_group_number": raw_sg_number,
+ "raw_space_group_operators_xyz": preserved_raw_operators,
+ "normalized_space_group_operators_xyz": sorted_runtime_ops,
+ "normalized_lookup_symbol": normalized_lookup_symbol,
+ "symmetry_normalization_policy": SYMMETRY_OPERATOR_NORMALIZATION_POLICY,
+ }
+
+
+def reconcile_phenix_rfactor_output(
+ phenix_payload: Mapping[str, Any],
+ input_ledger: Mapping[str, Any],
+ *,
+ expected_vendor_source_sha256: Mapping[str, str] | None = None,
+ expected_vendor_module_paths: Mapping[str, str | Path] | None = None,
+ expected_resolved_prefix: str | Path | None = None,
+ expected_phenix_version: str = toolchain.PHENIX_VERSION,
+ measured_build: Mapping[str, Any] | None = None,
+ phx_maps_mtz_path: Path | None = None,
+ phx_maps_mtz_bytes: bytes | None = None,
+ phx_obs_pair_only_mtz_path: Path | None = None,
+ phx_obs_pair_only_mtz_bytes: bytes | None = None,
+ phenix_diagnostics_path: Path | None = None,
+ phenix_diagnostics_payload: Mapping[str, Any] | None = None,
+ cache_manifest_artifacts: Mapping[str, Any] | None = None,
+) -> dict[str, Any]:
+ """Validate and reconcile the full-precision PHENIX helper payload against input ledger.
+
+ Enforces:
+ - Exact PHENIX version pin (``2.0-5936``), installation prefix, and runtime
+ vendor module provenance under the admitted ``resolved_prefix``.
+ - Recomputed canonical SHA-256 digests and zero per-scatterer diffs from the
+ full retained ``before``, ``after_model``, and ``after_fmodel`` scatterer
+ snapshots.
+ - Retained ``extract_xtal_data``, ``mmtbx.model.manager``, and
+ ``fmodel.update_all_scales`` diagnostic logs.
+ - Exact 1-to-1 HKL tracking and per-HKL ``f_obs`` / ``sigf_obs`` / ``is_free``
+ agreement across ``extracted_pre_fmodel``, ``final_effective``,
+ ``actual_work``, and ``actual_free``.
+ - Full-precision float64 replay of ``r_work``, ``r_free``, ``r_all``, gap,
+ and reported reflection counts and numerator/denominator sums.
+ - Stage-by-stage ``d_max`` and ``d_min`` resolution extrema recomputed from
+ each stage's reconciled reflection HKLs and the admitted input unit cell
+ via the general crystallographic reciprocal metric tensor.
+ - Internal consistency of ``2mFo-DFc`` and ``mFo-DFc`` JSON map coefficients
+ and binary verification of ``phx_maps.mtz`` and ``PHX_OBS_PAIR_ONLY.mtz``.
+ """
+ if not isinstance(phenix_payload, Mapping):
+ raise ValueError("phenix_payload must be a mapping")
+ if not isinstance(input_ledger, Mapping):
+ raise ValueError("input_ledger must be a mapping")
+
+ eligible_by_hkl = input_ledger.get("eligible_by_hkl")
+ raw_rows = input_ledger.get("raw_rows")
+ input_context = input_ledger.get("crystallographic_context")
+ if not isinstance(eligible_by_hkl, Mapping) or not eligible_by_hkl:
+ raise ValueError("input_ledger missing non-empty eligible_by_hkl mapping")
+ if not isinstance(raw_rows, list) or not raw_rows:
+ raise ValueError("input_ledger missing non-empty raw_rows list")
+ if not isinstance(input_context, Mapping):
+ raise ValueError("input_ledger missing crystallographic_context")
+ validated_input_cell = validate_unit_cell_parameters(
+ input_context.get("cell_a_b_c_alpha_beta_gamma"),
+ label="input_ledger.crystallographic_context.cell_a_b_c_alpha_beta_gamma",
+ )
+
+ raw_by_hkl: dict[str, Mapping[str, Any]] = {}
+ for row in raw_rows:
+ key = _hkl_key(row["hkl"])
+ if key in raw_by_hkl:
+ raise ValueError(f"Duplicate HKL {key} in input_ledger raw_rows")
+ raw_by_hkl[key] = row
+
+ # 1. Verify PHENIX version pin, prefix binding, and runtime module provenance.
+ if (
+ phenix_payload.get("artifact_role")
+ == "partial_unverified_phenix_evidence"
+ or phenix_payload.get("reusable_cache_output") is False
+ or phenix_payload.get("canonical_eval_qds_eligible") is False
+ ):
+ raise ValueError(
+ "Partial/unverified PHENIX evidence artifact cannot be reconciled "
+ "as a completed payload or admitted into canonical EVAL/QDS"
+ )
+ if expected_phenix_version != toolchain.PHENIX_VERSION:
+ raise ValueError(
+ f"expected_phenix_version {expected_phenix_version!r} does not match "
+ f"repository pin {toolchain.PHENIX_VERSION!r}"
+ )
+ if measured_build is not None:
+ validated_build = validate_measured_phenix_build(
+ measured_build, expected_version=expected_phenix_version
+ )
+ if expected_resolved_prefix is None:
+ expected_resolved_prefix = validated_build["resolved_prefix"]
+ elif str(Path(expected_resolved_prefix)) != validated_build["resolved_prefix"]:
+ raise ValueError(
+ f"expected_resolved_prefix {expected_resolved_prefix!r} disagrees with "
+ f"measured_build resolved_prefix {validated_build['resolved_prefix']!r}"
+ )
+
+ payload_version = phenix_payload.get("expected_phenix_version")
+ if payload_version != expected_phenix_version:
+ raise ValueError(
+ f"phenix_payload expected_phenix_version {payload_version!r} != "
+ f"{expected_phenix_version!r}"
+ )
+ payload_prefix = phenix_payload.get("expected_resolved_prefix")
+ if (
+ not isinstance(payload_prefix, str)
+ or not payload_prefix
+ or not Path(payload_prefix).is_absolute()
+ ):
+ raise ValueError(
+ f"phenix_payload missing absolute expected_resolved_prefix: {payload_prefix!r}"
+ )
+
+ runtime_env = phenix_payload.get("runtime_environment")
+ if not isinstance(runtime_env, Mapping):
+ raise ValueError("phenix_payload missing runtime_environment")
+ sys_prefix = runtime_env.get("sys_prefix")
+ resolved_sys_prefix = runtime_env.get("resolved_sys_prefix")
+ sys_executable = runtime_env.get("sys_executable")
+ resolved_sys_executable = runtime_env.get("resolved_sys_executable", sys_executable)
+ if (
+ not isinstance(sys_prefix, str)
+ or not sys_prefix
+ or not isinstance(resolved_sys_prefix, str)
+ or not resolved_sys_prefix
+ ):
+ raise ValueError("runtime_environment missing sys_prefix / resolved_sys_prefix")
+ prefix_path = Path(resolved_sys_prefix)
+ if not prefix_path.is_absolute():
+ raise ValueError(f"resolved_sys_prefix must be absolute: {resolved_sys_prefix}")
+ if resolved_sys_prefix != str(Path(payload_prefix)):
+ raise ValueError(
+ f"runtime_environment.resolved_sys_prefix {resolved_sys_prefix!r} != "
+ f"payload expected_resolved_prefix {payload_prefix!r}"
+ )
+ if expected_resolved_prefix is not None and resolved_sys_prefix != str(
+ Path(expected_resolved_prefix)
+ ):
+ raise ValueError(
+ f"runtime_environment.resolved_sys_prefix {resolved_sys_prefix!r} != "
+ f"expected_resolved_prefix {str(Path(expected_resolved_prefix))!r}"
+ )
+ for exec_label, exec_val in (
+ ("sys_executable", sys_executable),
+ ("resolved_sys_executable", resolved_sys_executable),
+ ):
+ if (
+ not isinstance(exec_val, str)
+ or not exec_val
+ or not Path(exec_val).is_absolute()
+ or not Path(exec_val).is_relative_to(prefix_path)
+ ):
+ raise ValueError(
+ f"runtime_environment.{exec_label} {exec_val!r} is not inside "
+ f"resolved_sys_prefix {prefix_path}"
+ )
+
+ modules = runtime_env.get("vendor_modules")
+ if not isinstance(modules, Mapping) or set(modules) != set(
+ PHENIX_VENDOR_SOURCE_RELPATHS
+ ):
+ raise ValueError(
+ f"runtime_environment.vendor_modules keys must match "
+ f"{sorted(PHENIX_VENDOR_SOURCE_RELPATHS)}"
+ )
+ for mod_key, mod_info in modules.items():
+ if not isinstance(mod_info, Mapping):
+ raise ValueError(f"vendor_modules[{mod_key!r}] must be a mapping")
+ mod_path_str = mod_info.get("resolved_path")
+ mod_sha = mod_info.get("sha256")
+ if not isinstance(mod_path_str, str) or not mod_path_str:
+ raise ValueError(f"vendor_modules[{mod_key!r}] missing resolved_path")
+ if not isinstance(mod_sha, str) or len(mod_sha) != 64:
+ raise ValueError(f"vendor_modules[{mod_key!r}] missing valid sha256")
+ mod_path = Path(mod_path_str)
+ if not mod_path.is_absolute() or not mod_path.is_relative_to(prefix_path):
+ raise ValueError(
+ f"Vendor module {mod_key} at {mod_path} escapes resolved_sys_prefix "
+ f"{prefix_path}"
+ )
+ rel_parts = mod_path.relative_to(prefix_path).parts
+ expected_rel = PHENIX_VENDOR_SOURCE_RELPATHS[mod_key]
+ if (
+ len(rel_parts) != 3 + len(expected_rel.parts)
+ or rel_parts[0] != "lib"
+ or not rel_parts[1].startswith("python")
+ or rel_parts[2] != "site-packages"
+ or Path(*rel_parts[3:]) != expected_rel
+ ):
+ raise ValueError(
+ f"Vendor module {mod_key} path {mod_path} does not match canonical "
+ f"lib/python*/site-packages/{expected_rel} under {prefix_path}"
+ )
+ if expected_vendor_module_paths is not None:
+ expected_mod_path = expected_vendor_module_paths.get(mod_key)
+ if expected_mod_path is None or str(mod_path) != str(
+ Path(expected_mod_path)
+ ):
+ raise ValueError(
+ f"Vendor module {mod_key} resolved_path mismatch: expected "
+ f"{expected_mod_path}, runtime loaded {mod_path}"
+ )
+ if expected_vendor_source_sha256 is not None:
+ expected_sha = expected_vendor_source_sha256.get(mod_key)
+ if expected_sha != mod_sha:
+ raise ValueError(
+ f"Vendor module {mod_key} SHA-256 mismatch: expected "
+ f"{expected_sha}, runtime loaded {mod_sha}"
+ )
+
+ # 2. Verify extraction parameters, absence of second-block cutoff, and retained logs.
+ extraction_info = phenix_payload.get("extraction")
+ if not isinstance(extraction_info, Mapping):
+ raise ValueError("phenix_payload missing extraction section")
+ if extraction_info.get("labels") != ["F-obs", "SIGF-obs"]:
+ raise ValueError(
+ f"Unexpected extraction labels: {extraction_info.get('labels')!r}"
+ )
+ if extraction_info.get("r_free_flags_label") != "R-free-flags":
+ raise ValueError(
+ f"Unexpected R-free label: {extraction_info.get('r_free_flags_label')!r}"
+ )
+ if (
+ type(extraction_info.get("test_flag_value")) is not int
+ or extraction_info.get("test_flag_value") != FREE_FLAG_VALUE
+ ):
+ raise ValueError(
+ f"Expected test_flag_value == {FREE_FLAG_VALUE}, got "
+ f"{extraction_info.get('test_flag_value')!r}"
+ )
+ if extraction_info.get("keep_going") is not False:
+ raise ValueError("extract_xtal_data must run with keep_going=False")
+ if extraction_info.get("err"):
+ raise ValueError(
+ f"extract_xtal_data reported errors: {extraction_info.get('err')!r}"
+ )
+ if extraction_info.get("applied_pdb_header_resolution_cut") is not False:
+ raise ValueError(
+ "Second-block PDB header resolution cutoff must not be applied"
+ )
+ if not isinstance(extraction_info.get("log_text"), str):
+ raise ValueError("extraction.log_text must be a string")
+
+ diagnostics = phenix_payload.get("diagnostics")
+ if not isinstance(diagnostics, Mapping):
+ raise ValueError("phenix_payload missing diagnostics section")
+ for log_key in (
+ "extract_xtal_data_log",
+ "model_manager_log",
+ "fmodel_update_all_scales_log",
+ ):
+ if not isinstance(diagnostics.get(log_key), str):
+ raise ValueError(f"diagnostics[{log_key!r}] must be a string")
+ if diagnostics.get("failure") is not None:
+ raise ValueError(
+ f"diagnostics.failure must be None on success, got {diagnostics.get('failure')!r}"
+ )
+ if diagnostics["extract_xtal_data_log"] != extraction_info["log_text"]:
+ raise ValueError(
+ "diagnostics.extract_xtal_data_log disagrees with extraction.log_text"
+ )
+
+ if phenix_diagnostics_path is not None or phenix_diagnostics_payload is not None:
+ if phenix_diagnostics_path is not None:
+ diag_sha = _sha256_regular_file(Path(phenix_diagnostics_path))
+ if (
+ cache_manifest_artifacts is not None
+ and "phenix_diagnostics_json" in cache_manifest_artifacts
+ ):
+ expected_diag_sha = cache_manifest_artifacts["phenix_diagnostics_json"][
+ "sha256"
+ ]
+ if diag_sha != expected_diag_sha:
+ raise ValueError(
+ f"phenix_diagnostics.json SHA-256 {diag_sha} != "
+ f"manifest {expected_diag_sha}"
+ )
+ loaded_diag = json.loads(
+ Path(phenix_diagnostics_path).read_text(encoding="utf-8")
+ )
+ if (
+ phenix_diagnostics_payload is not None
+ and dict(loaded_diag) != dict(phenix_diagnostics_payload)
+ ):
+ raise ValueError(
+ "phenix_diagnostics_path content disagrees with phenix_diagnostics_payload"
+ )
+ diag_doc: Mapping[str, Any] = loaded_diag
+ else:
+ assert phenix_diagnostics_payload is not None
+ diag_doc = phenix_diagnostics_payload
+ if (
+ not isinstance(diag_doc, Mapping)
+ or diag_doc.get("status") != "succeeded"
+ or diag_doc.get("failure") is not None
+ or diag_doc.get("extract_xtal_data_log")
+ != diagnostics["extract_xtal_data_log"]
+ or diag_doc.get("model_manager_log") != diagnostics["model_manager_log"]
+ or diag_doc.get("fmodel_update_all_scales_log")
+ != diagnostics["fmodel_update_all_scales_log"]
+ ):
+ raise ValueError(
+ "Standalone phenix_diagnostics.json disagrees with phenix_payload.diagnostics"
+ )
+
+ # 3. Recompute canonical scatterer snapshot digests and verify zero mutation.
+ xrs_check = phenix_payload.get("xray_structure_invariance")
+ if not isinstance(xrs_check, Mapping):
+ raise ValueError("phenix_payload missing xray_structure_invariance")
+ snapshots = xrs_check.get("snapshots")
+ if not isinstance(snapshots, Mapping) or set(snapshots) != {
+ "before",
+ "after_model",
+ "after_fmodel",
+ }:
+ raise ValueError(
+ "xray_structure_invariance must contain snapshots.{before,after_model,after_fmodel}"
+ )
+ recomputed_xrs = onesar_phenix_measure.compare_scatterer_snapshots(
+ snapshots["before"],
+ snapshots["after_model"],
+ snapshots["after_fmodel"],
+ )
+ if (
+ type(xrs_check.get("n_scatterers")) is not int
+ or xrs_check["n_scatterers"] <= 0
+ or xrs_check["n_scatterers"] != recomputed_xrs["n_scatterers"]
+ ):
+ raise ValueError(
+ "xray_structure_invariance n_scatterers disagrees with recomputed snapshots"
+ )
+ before_sha = xrs_check.get("before_sha256")
+ after_model_sha = xrs_check.get("after_model_sha256")
+ after_fmodel_sha = xrs_check.get("after_fmodel_sha256")
+ if (
+ before_sha != recomputed_xrs["before_sha256"]
+ or after_model_sha != recomputed_xrs["after_model_sha256"]
+ or after_fmodel_sha != recomputed_xrs["after_fmodel_sha256"]
+ ):
+ raise ValueError(
+ "xray_structure_invariance snapshot SHA-256 fields disagree with "
+ "recomputed scatterer records"
+ )
+ if (
+ not isinstance(before_sha, str)
+ or not before_sha
+ or before_sha != after_model_sha
+ or before_sha != after_fmodel_sha
+ or xrs_check.get("identical_before_vs_after_model") is not True
+ or xrs_check.get("identical_before_vs_after_fmodel") is not True
+ or recomputed_xrs["identical_before_vs_after_model"] is not True
+ or recomputed_xrs["identical_before_vs_after_fmodel"] is not True
+ ):
+ raise ValueError(
+ "Model xray_structure mutated during f_model construction or scaling: "
+ f"before={before_sha!r}, after_model={after_model_sha!r}, "
+ f"after_fmodel={after_fmodel_sha!r}"
+ )
+ for diff_key in (
+ "max_abs_site_cart_diff",
+ "max_abs_occupancy_diff",
+ "max_abs_u_iso_diff",
+ "max_abs_u_star_diff",
+ ):
+ diff_val = _verify_finite_nonnegative(xrs_check.get(diff_key), diff_key)
+ recomputed_diff = float(recomputed_xrs[diff_key])
+ if diff_val != 0.0 or recomputed_diff != 0.0:
+ raise ValueError(
+ f"Non-zero {diff_key} across scaling: reported={diff_val}, "
+ f"recomputed={recomputed_diff}"
+ )
+
+ # 4. Reconcile extracted pre-fmodel and final effective reflection sets.
+ extracted_rows = phenix_payload.get("extracted_pre_fmodel")
+ final_rows = phenix_payload.get("final_effective")
+ work_rows = phenix_payload.get("actual_work")
+ free_rows = phenix_payload.get("actual_free")
+ if (
+ not isinstance(extracted_rows, list)
+ or not isinstance(final_rows, list)
+ or not isinstance(work_rows, list)
+ or not isinstance(free_rows, list)
+ ):
+ raise ValueError(
+ "phenix_payload must contain list arrays for extracted_pre_fmodel, "
+ "final_effective, actual_work, and actual_free"
+ )
+ if not extracted_rows or not final_rows or not work_rows or not free_rows:
+ raise ValueError(
+ "extracted_pre_fmodel, final_effective, actual_work, and actual_free "
+ "must all be non-empty"
+ )
+
+ extracted_by_hkl: dict[str, Mapping[str, Any]] = {}
+ for entry in extracted_rows:
+ key = _hkl_key(entry.get("hkl"))
+ if key in extracted_by_hkl:
+ raise ValueError(
+ f"Duplicate HKL {key} in extracted_pre_fmodel (many-to-one collision)"
+ )
+ if key not in raw_by_hkl:
+ raise ValueError(
+ f"Extracted HKL {key} is absent from input MTZ raw ledger "
+ "(unexpected symmetry/reindexing transformation)"
+ )
+ if key not in eligible_by_hkl:
+ raise ValueError(
+ f"Extracted HKL {key} was not input-eligible in input MTZ ledger"
+ )
+ f_obs_val = _verify_finite_positive(
+ entry.get("f_obs"), f"extracted_pre_fmodel[{key}].f_obs"
+ )
+ sigf_val = _verify_finite_positive(
+ entry.get("sigf_obs"), f"extracted_pre_fmodel[{key}].sigf_obs"
+ )
+ is_free = entry.get("is_free")
+ if type(is_free) is not bool:
+ raise ValueError(f"extracted_pre_fmodel[{key}].is_free must be bool")
+ orig_entry = eligible_by_hkl[key]
+ if is_free != bool(orig_entry["is_free"]):
+ raise ValueError(
+ f"Flag mismatch at extracted HKL {key}: PHENIX is_free={is_free}, "
+ f"input ledger is_free={orig_entry['is_free']}"
+ )
+ if not math.isclose(
+ f_obs_val, float(orig_entry["f_obs"]), rel_tol=0.0, abs_tol=1e-6
+ ) or not math.isclose(
+ sigf_val, float(orig_entry["sigf_obs"]), rel_tol=0.0, abs_tol=1e-6
+ ):
+ raise ValueError(
+ f"Pre-scaling extracted F/SIGF drifted from input MTZ at {key}: "
+ f"extracted=({f_obs_val}, {sigf_val}), "
+ f"input=({orig_entry['f_obs']}, {orig_entry['sigf_obs']})"
+ )
+ extracted_by_hkl[key] = entry
+
+ final_by_hkl: dict[str, Mapping[str, Any]] = {}
+ for entry in final_rows:
+ key = _hkl_key(entry.get("hkl"))
+ if key in final_by_hkl:
+ raise ValueError(f"Duplicate HKL {key} in final_effective")
+ if key not in extracted_by_hkl:
+ raise ValueError(
+ f"Final effective HKL {key} was not in extracted_pre_fmodel"
+ )
+ f_final = _verify_finite_positive(
+ entry.get("f_obs"), f"final_effective[{key}].f_obs"
+ )
+ sigf_final = _verify_finite_positive(
+ entry.get("sigf_obs"), f"final_effective[{key}].sigf_obs"
+ )
+ is_free = entry.get("is_free")
+ if type(is_free) is not bool or is_free != bool(eligible_by_hkl[key]["is_free"]):
+ raise ValueError(
+ f"Flag mismatch at final_effective HKL {key}: {is_free!r}"
+ )
+ final_by_hkl[key] = {
+ "hkl": list(entry["hkl"]),
+ "f_obs": f_final,
+ "sigf_obs": sigf_final,
+ "is_free": is_free,
+ }
+
+ # 5. Reconcile actual_work and actual_free sets and replay R_work / R_free / R_all in float64.
+ scales = phenix_payload.get("scales_and_rfactors")
+ if not isinstance(scales, Mapping):
+ raise ValueError("phenix_payload missing scales_and_rfactors")
+ scale_k1_w = _verify_finite_positive(scales.get("scale_k1_w"), "scale_k1_w")
+ scale_k1_t = _verify_finite_positive(scales.get("scale_k1_t"), "scale_k1_t")
+ scale_k1 = _verify_finite_positive(scales.get("scale_k1"), "scale_k1")
+ reported_r_work = _verify_finite_nonnegative(scales.get("r_work"), "r_work")
+ reported_r_free = _verify_finite_nonnegative(scales.get("r_free"), "r_free")
+ reported_r_all = _verify_finite_nonnegative(scales.get("r_all"), "r_all")
+ reported_gap = _verify_finite(
+ scales.get("r_free_r_work_gap"), "r_free_r_work_gap"
+ )
+ reported_extracted_count = _verify_int_count(
+ scales.get("extracted_pre_fmodel_count"),
+ "scales_and_rfactors.extracted_pre_fmodel_count",
+ )
+ reported_final_count = _verify_int_count(
+ scales.get("final_effective_count"),
+ "scales_and_rfactors.final_effective_count",
+ )
+ reported_work_count = _verify_int_count(
+ scales.get("work_count"), "scales_and_rfactors.work_count"
+ )
+ reported_free_count = _verify_int_count(
+ scales.get("free_count"), "scales_and_rfactors.free_count"
+ )
+ reported_work_num = _verify_finite_nonnegative(
+ scales.get("work_numerator"), "scales_and_rfactors.work_numerator"
+ )
+ reported_work_den = _verify_finite_positive(
+ scales.get("work_denominator"), "scales_and_rfactors.work_denominator"
+ )
+ reported_free_num = _verify_finite_nonnegative(
+ scales.get("free_numerator"), "scales_and_rfactors.free_numerator"
+ )
+ reported_free_den = _verify_finite_positive(
+ scales.get("free_denominator"), "scales_and_rfactors.free_denominator"
+ )
+
+ work_by_hkl: dict[str, dict[str, Any]] = {}
+ work_num = 0.0
+ work_den = 0.0
+ all_num = 0.0
+ all_den = 0.0
+ max_f_obs_scale_diff = 0.0
+
+ for entry in work_rows:
+ key = _hkl_key(entry.get("hkl"))
+ if key in work_by_hkl:
+ raise ValueError(f"Duplicate HKL {key} in actual_work")
+ if key not in final_by_hkl:
+ raise ValueError(f"actual_work HKL {key} not in final_effective")
+ orig = eligible_by_hkl[key]
+ if orig["is_free"] or orig["r_free_flag"] != WORK_FLAG_VALUE:
+ raise ValueError(
+ f"actual_work HKL {key} has free flag {orig['r_free_flag']} in input MTZ"
+ )
+ final_entry = final_by_hkl[key]
+ if final_entry["is_free"] is not False:
+ raise ValueError(
+ f"actual_work HKL {key} has is_free={final_entry['is_free']!r} in final_effective"
+ )
+ fo = _verify_finite_positive(entry.get("f_obs"), f"actual_work[{key}].f_obs")
+ sigfo = _verify_finite_positive(
+ entry.get("sigf_obs"), f"actual_work[{key}].sigf_obs"
+ )
+ if not math.isclose(
+ fo, float(final_entry["f_obs"]), rel_tol=1e-15, abs_tol=1e-15
+ ) or not math.isclose(
+ sigfo, float(final_entry["sigf_obs"]), rel_tol=1e-15, abs_tol=1e-15
+ ):
+ raise ValueError(
+ f"actual_work[{key}] F/SIGF ({fo}, {sigfo}) disagrees with "
+ f"final_effective[{key}] ({final_entry['f_obs']}, {final_entry['sigf_obs']})"
+ )
+ fm_raw_re = _verify_finite(
+ entry.get("f_model_real"), f"actual_work[{key}].f_model_real"
+ )
+ fm_raw_im = _verify_finite(
+ entry.get("f_model_imag"), f"actual_work[{key}].f_model_imag"
+ )
+ fm_raw_amp = _verify_finite_nonnegative(
+ entry.get("f_model_amplitude"), f"actual_work[{key}].f_model_amplitude"
+ )
+ fm_scaled_re = _verify_finite(
+ entry.get("f_model_scaled_with_k1_w_real"),
+ f"actual_work[{key}].f_model_scaled_with_k1_w_real",
+ )
+ fm_scaled_im = _verify_finite(
+ entry.get("f_model_scaled_with_k1_w_imag"),
+ f"actual_work[{key}].f_model_scaled_with_k1_w_imag",
+ )
+ fm_scaled_amp = _verify_finite_nonnegative(
+ entry.get("f_model_scaled_with_k1_w_amplitude"),
+ f"actual_work[{key}].f_model_scaled_with_k1_w_amplitude",
+ )
+ calc_raw_amp = math.hypot(fm_raw_re, fm_raw_im)
+ calc_scaled_amp = math.hypot(fm_scaled_re, fm_scaled_im)
+ if not math.isclose(calc_raw_amp, fm_raw_amp, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"actual_work[{key}] raw amplitude {fm_raw_amp} != hypot {calc_raw_amp}"
+ )
+ if not math.isclose(
+ calc_scaled_amp, fm_scaled_amp, rel_tol=1e-12, abs_tol=1e-12
+ ):
+ raise ValueError(
+ f"actual_work[{key}] scaled amplitude {fm_scaled_amp} != hypot "
+ f"{calc_scaled_amp}"
+ )
+ if not math.isclose(
+ fm_scaled_amp, scale_k1_w * fm_raw_amp, rel_tol=1e-12, abs_tol=1e-12
+ ):
+ raise ValueError(
+ f"actual_work[{key}] f_model_scaled_with_k1_w amplitude {fm_scaled_amp} "
+ f"disagrees with scale_k1_w * |f_model_work| = {scale_k1_w * fm_raw_amp}"
+ )
+ if not math.isclose(
+ fm_scaled_re, scale_k1_w * fm_raw_re, rel_tol=1e-12, abs_tol=1e-12
+ ) or not math.isclose(
+ fm_scaled_im, scale_k1_w * fm_raw_im, rel_tol=1e-12, abs_tol=1e-12
+ ):
+ raise ValueError(
+ f"actual_work[{key}] f_model_scaled_with_k1_w complex components "
+ f"({fm_scaled_re}, {fm_scaled_im}) disagree with "
+ f"scale_k1_w * f_model_work = "
+ f"({scale_k1_w * fm_raw_re}, {scale_k1_w * fm_raw_im})"
+ )
+
+ work_num += abs(fo - fm_scaled_amp)
+ work_den += fo
+ all_num += abs(fo - scale_k1 * fm_raw_amp)
+ all_den += fo
+ max_f_obs_scale_diff = max(
+ max_f_obs_scale_diff, abs(fo - float(orig["f_obs"]))
+ )
+ work_by_hkl[key] = {
+ "hkl": list(entry["hkl"]),
+ "original_f_obs": float(orig["f_obs"]),
+ "original_sigf_obs": float(orig["sigf_obs"]),
+ "final_f_obs": fo,
+ "final_sigf_obs": sigfo,
+ "r_free_flag": WORK_FLAG_VALUE,
+ "f_model_amplitude": fm_raw_amp,
+ "f_model_scaled_with_k1_w_amplitude": fm_scaled_amp,
+ }
+
+ free_by_hkl: dict[str, dict[str, Any]] = {}
+ free_num = 0.0
+ free_den = 0.0
+
+ for entry in free_rows:
+ key = _hkl_key(entry.get("hkl"))
+ if key in free_by_hkl:
+ raise ValueError(f"Duplicate HKL {key} in actual_free")
+ if key in work_by_hkl:
+ raise ValueError(
+ f"HKL {key} appears in both actual_work and actual_free (non-disjoint)"
+ )
+ if key not in final_by_hkl:
+ raise ValueError(f"actual_free HKL {key} not in final_effective")
+ orig = eligible_by_hkl[key]
+ if not orig["is_free"] or orig["r_free_flag"] != FREE_FLAG_VALUE:
+ raise ValueError(
+ f"actual_free HKL {key} has work flag {orig['r_free_flag']} in input MTZ"
+ )
+ final_entry = final_by_hkl[key]
+ if final_entry["is_free"] is not True:
+ raise ValueError(
+ f"actual_free HKL {key} has is_free={final_entry['is_free']!r} in final_effective"
+ )
+ fo = _verify_finite_positive(entry.get("f_obs"), f"actual_free[{key}].f_obs")
+ sigfo = _verify_finite_positive(
+ entry.get("sigf_obs"), f"actual_free[{key}].sigf_obs"
+ )
+ if not math.isclose(
+ fo, float(final_entry["f_obs"]), rel_tol=1e-15, abs_tol=1e-15
+ ) or not math.isclose(
+ sigfo, float(final_entry["sigf_obs"]), rel_tol=1e-15, abs_tol=1e-15
+ ):
+ raise ValueError(
+ f"actual_free[{key}] F/SIGF ({fo}, {sigfo}) disagrees with "
+ f"final_effective[{key}] ({final_entry['f_obs']}, {final_entry['sigf_obs']})"
+ )
+ fm_raw_re = _verify_finite(
+ entry.get("f_model_real"), f"actual_free[{key}].f_model_real"
+ )
+ fm_raw_im = _verify_finite(
+ entry.get("f_model_imag"), f"actual_free[{key}].f_model_imag"
+ )
+ fm_raw_amp = _verify_finite_nonnegative(
+ entry.get("f_model_amplitude"), f"actual_free[{key}].f_model_amplitude"
+ )
+ fm_scaled_re = _verify_finite(
+ entry.get("f_model_scaled_with_k1_t_real"),
+ f"actual_free[{key}].f_model_scaled_with_k1_t_real",
+ )
+ fm_scaled_im = _verify_finite(
+ entry.get("f_model_scaled_with_k1_t_imag"),
+ f"actual_free[{key}].f_model_scaled_with_k1_t_imag",
+ )
+ fm_scaled_amp = _verify_finite_nonnegative(
+ entry.get("f_model_scaled_with_k1_t_amplitude"),
+ f"actual_free[{key}].f_model_scaled_with_k1_t_amplitude",
+ )
+ calc_raw_amp = math.hypot(fm_raw_re, fm_raw_im)
+ calc_scaled_amp = math.hypot(fm_scaled_re, fm_scaled_im)
+ if not math.isclose(calc_raw_amp, fm_raw_amp, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"actual_free[{key}] raw amplitude {fm_raw_amp} != hypot {calc_raw_amp}"
+ )
+ if not math.isclose(
+ calc_scaled_amp, fm_scaled_amp, rel_tol=1e-12, abs_tol=1e-12
+ ):
+ raise ValueError(
+ f"actual_free[{key}] scaled amplitude {fm_scaled_amp} != hypot "
+ f"{calc_scaled_amp}"
+ )
+ if not math.isclose(
+ fm_scaled_amp, scale_k1_t * fm_raw_amp, rel_tol=1e-12, abs_tol=1e-12
+ ):
+ raise ValueError(
+ f"actual_free[{key}] f_model_scaled_with_k1_t amplitude {fm_scaled_amp} "
+ f"disagrees with scale_k1_t * |f_model_free| = {scale_k1_t * fm_raw_amp}"
+ )
+ if not math.isclose(
+ fm_scaled_re, scale_k1_t * fm_raw_re, rel_tol=1e-12, abs_tol=1e-12
+ ) or not math.isclose(
+ fm_scaled_im, scale_k1_t * fm_raw_im, rel_tol=1e-12, abs_tol=1e-12
+ ):
+ raise ValueError(
+ f"actual_free[{key}] f_model_scaled_with_k1_t complex components "
+ f"({fm_scaled_re}, {fm_scaled_im}) disagree with "
+ f"scale_k1_t * f_model_free = "
+ f"({scale_k1_t * fm_raw_re}, {scale_k1_t * fm_raw_im})"
+ )
+
+ free_num += abs(fo - fm_scaled_amp)
+ free_den += fo
+ all_num += abs(fo - scale_k1 * fm_raw_amp)
+ all_den += fo
+ max_f_obs_scale_diff = max(
+ max_f_obs_scale_diff, abs(fo - float(orig["f_obs"]))
+ )
+ free_by_hkl[key] = {
+ "hkl": list(entry["hkl"]),
+ "original_f_obs": float(orig["f_obs"]),
+ "original_sigf_obs": float(orig["sigf_obs"]),
+ "final_f_obs": fo,
+ "final_sigf_obs": sigfo,
+ "r_free_flag": FREE_FLAG_VALUE,
+ "f_model_amplitude": fm_raw_amp,
+ "f_model_scaled_with_k1_t_amplitude": fm_scaled_amp,
+ }
+
+ if set(work_by_hkl) | set(free_by_hkl) != set(final_by_hkl):
+ raise ValueError(
+ "Union of actual_work and actual_free HKLs does not equal final_effective"
+ )
+ if (
+ reported_extracted_count != len(extracted_by_hkl)
+ or reported_final_count != len(final_by_hkl)
+ or reported_work_count != len(work_by_hkl)
+ or reported_free_count != len(free_by_hkl)
+ ):
+ raise ValueError(
+ "Reported scales_and_rfactors counts disagree with reflection arrays: "
+ f"reported=({reported_extracted_count}, {reported_final_count}, "
+ f"{reported_work_count}, {reported_free_count}), "
+ f"actual=({len(extracted_by_hkl)}, {len(final_by_hkl)}, "
+ f"{len(work_by_hkl)}, {len(free_by_hkl)})"
+ )
+ if work_den <= 0.0 or free_den <= 0.0 or all_den <= 0.0:
+ raise ValueError("Work, free, and all denominators must be strictly positive")
+
+ if not math.isclose(reported_work_num, work_num, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"Reported work_numerator {reported_work_num} != recomputed {work_num}"
+ )
+ if not math.isclose(reported_work_den, work_den, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"Reported work_denominator {reported_work_den} != recomputed {work_den}"
+ )
+ if not math.isclose(reported_free_num, free_num, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"Reported free_numerator {reported_free_num} != recomputed {free_num}"
+ )
+ if not math.isclose(reported_free_den, free_den, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"Reported free_denominator {reported_free_den} != recomputed {free_den}"
+ )
+ if "all_numerator" in scales:
+ reported_all_num = _verify_finite_nonnegative(
+ scales.get("all_numerator"), "scales_and_rfactors.all_numerator"
+ )
+ if not math.isclose(reported_all_num, all_num, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"Reported all_numerator {reported_all_num} != recomputed {all_num}"
+ )
+ if "all_denominator" in scales:
+ reported_all_den = _verify_finite_positive(
+ scales.get("all_denominator"), "scales_and_rfactors.all_denominator"
+ )
+ if not math.isclose(reported_all_den, all_den, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"Reported all_denominator {reported_all_den} != recomputed {all_den}"
+ )
+
+ replayed_r_work = work_num / work_den
+ replayed_r_free = free_num / free_den
+ replayed_r_all = all_num / all_den
+ replayed_gap = replayed_r_free - replayed_r_work
+
+ if not math.isclose(replayed_r_work, reported_r_work, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"Replayed r_work {replayed_r_work:.15f} disagrees with native "
+ f"fmodel.r_work() {reported_r_work:.15f}"
+ )
+ if not math.isclose(replayed_r_free, reported_r_free, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"Replayed r_free {replayed_r_free:.15f} disagrees with native "
+ f"fmodel.r_free() {reported_r_free:.15f}"
+ )
+ if not math.isclose(replayed_r_all, reported_r_all, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"Replayed r_all {replayed_r_all:.15f} disagrees with native "
+ f"fmodel.r_all() {reported_r_all:.15f}"
+ )
+ if not math.isclose(replayed_gap, reported_gap, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"Replayed r_free - r_work gap {replayed_gap:.15f} disagrees with "
+ f"reported {reported_gap:.15f}"
+ )
+
+ # 5b. Validate retained runtime unit cell against installed CCP4/iotbx float32
+ # conversion of the admitted input cell, then replay stage resolution ranges (#1020).
+ expected_runtime_float32_cell = ccp4_mtz_float32_unit_cell(
+ validated_input_cell,
+ label="input_ledger.crystallographic_context.cell_a_b_c_alpha_beta_gamma",
+ )
+ crystal_symmetry = phenix_payload.get("crystal_symmetry")
+ if not isinstance(crystal_symmetry, Mapping):
+ raise ValueError("phenix_payload missing crystal_symmetry section")
+ space_group_symbol = crystal_symmetry.get("space_group_symbol")
+ if not isinstance(space_group_symbol, str) or not space_group_symbol.strip():
+ raise ValueError(
+ "phenix_payload.crystal_symmetry.space_group_symbol must be a non-empty string"
+ )
+ raw_runtime_cell = crystal_symmetry.get("unit_cell")
+ if not isinstance(raw_runtime_cell, (list, tuple)) or len(raw_runtime_cell) != 6:
+ raise ValueError(
+ "phenix_payload.crystal_symmetry.unit_cell must be a 6-element numeric sequence, "
+ f"got {raw_runtime_cell!r}"
+ )
+ for idx, val in enumerate(raw_runtime_cell):
+ _verify_finite_positive(
+ val, f"phenix_payload.crystal_symmetry.unit_cell[{idx}]"
+ )
+ validated_runtime_cell = validate_unit_cell_parameters(
+ raw_runtime_cell, label="phenix_payload.crystal_symmetry.unit_cell"
+ )
+ for idx, (param_name, rt_val, exp_val) in enumerate(
+ zip(
+ ("a", "b", "c", "alpha", "beta", "gamma"),
+ validated_runtime_cell,
+ expected_runtime_float32_cell,
+ )
+ ):
+ if not math.isclose(rt_val, exp_val, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"phenix_payload.crystal_symmetry.unit_cell[{idx}] ({param_name}) "
+ f"{rt_val:.15f} disagrees with installed CCP4/iotbx float32 "
+ f"conversion {exp_val:.15f} of admitted input cell "
+ f"{validated_input_cell[idx]:.15f}"
+ )
+ validated_runtime_space_group = validate_runtime_space_group(
+ crystal_symmetry,
+ input_context.get("symmetry"),
+ )
+
+ resolution_ranges = phenix_payload.get("resolution_ranges")
+ if not isinstance(resolution_ranges, Mapping) or set(
+ resolution_ranges
+ ) != set(EXPECTED_RESOLUTION_STAGES):
+ raise ValueError(
+ "phenix_payload.resolution_ranges must be a mapping with exact keys "
+ f"{list(EXPECTED_RESOLUTION_STAGES)!r}, got "
+ f"{list(resolution_ranges) if isinstance(resolution_ranges, Mapping) else resolution_ranges!r}"
+ )
+ stage_hkls_by_name: dict[str, list[Any]] = {
+ "extracted_pre_fmodel": [
+ entry["hkl"] for entry in extracted_by_hkl.values()
+ ],
+ "final_effective": [entry["hkl"] for entry in final_by_hkl.values()],
+ "actual_work": [entry["hkl"] for entry in work_by_hkl.values()],
+ "actual_free": [entry["hkl"] for entry in free_by_hkl.values()],
+ }
+ reconciled_resolution_ranges: dict[str, dict[str, float]] = {}
+ for stage_name in EXPECTED_RESOLUTION_STAGES:
+ stage_range = resolution_ranges.get(stage_name)
+ if not isinstance(stage_range, Mapping) or set(stage_range) != {
+ "d_max",
+ "d_min",
+ }:
+ raise ValueError(
+ f"resolution_ranges[{stage_name!r}] must be a mapping with exact keys "
+ f"['d_max', 'd_min'], got "
+ f"{list(stage_range) if isinstance(stage_range, Mapping) else stage_range!r}"
+ )
+ reported_d_max = _verify_finite_positive(
+ stage_range.get("d_max"), f"resolution_ranges[{stage_name!r}].d_max"
+ )
+ reported_d_min = _verify_finite_positive(
+ stage_range.get("d_min"), f"resolution_ranges[{stage_name!r}].d_min"
+ )
+ if reported_d_max < reported_d_min:
+ raise ValueError(
+ f"resolution_ranges[{stage_name!r}] has inverted bounds: "
+ f"d_max={reported_d_max} < d_min={reported_d_min}"
+ )
+ recomputed_range = compute_stage_resolution_range(
+ stage_hkls_by_name[stage_name],
+ validated_runtime_cell,
+ stage_label=stage_name,
+ )
+ recomputed_d_max = recomputed_range["d_max"]
+ recomputed_d_min = recomputed_range["d_min"]
+ if not math.isclose(
+ reported_d_max, recomputed_d_max, rel_tol=1e-12, abs_tol=1e-12
+ ):
+ raise ValueError(
+ f"resolution_ranges[{stage_name!r}].d_max {reported_d_max:.15f} "
+ f"disagrees with recomputed {recomputed_d_max:.15f}"
+ )
+ if not math.isclose(
+ reported_d_min, recomputed_d_min, rel_tol=1e-12, abs_tol=1e-12
+ ):
+ raise ValueError(
+ f"resolution_ranges[{stage_name!r}].d_min {reported_d_min:.15f} "
+ f"disagrees with recomputed {recomputed_d_min:.15f}"
+ )
+ reconciled_resolution_ranges[stage_name] = {
+ "d_max": reported_d_max,
+ "d_min": reported_d_min,
+ "recomputed_d_max": recomputed_d_max,
+ "recomputed_d_min": recomputed_d_min,
+ "d_max_abs_diff": abs(recomputed_d_max - reported_d_max),
+ "d_min_abs_diff": abs(recomputed_d_min - reported_d_min),
+ }
+
+ # 6. Validate fresh complex map-coefficient export metadata, JSON records, and MTZ files.
+ map_coeffs = phenix_payload.get("map_coefficients")
+ if not isinstance(map_coeffs, Mapping):
+ raise ValueError("phenix_payload missing map_coefficients section")
+ expected_maps = {
+ "2mFo-DFc": ("2FOFCWT", "PH2FOFCWT"),
+ "mFo-DFc": ("FOFCWT", "PHFOFCWT"),
+ }
+ map_summary: dict[str, Any] = {}
+ map_records_by_hkl: dict[str, dict[str, Mapping[str, Any]]] = {}
+ for map_key, (amp_label, phase_label) in expected_maps.items():
+ mdata = map_coeffs.get(map_key)
+ if not isinstance(mdata, Mapping):
+ raise ValueError(f"map_coefficients missing {map_key!r}")
+ if (
+ mdata.get("map_type") != map_key
+ or mdata.get("amplitude_label") != amp_label
+ or mdata.get("phase_label") != phase_label
+ ):
+ raise ValueError(f"Invalid labels for map_coefficients[{map_key!r}]")
+ params = mdata.get("parameters")
+ if not isinstance(params, Mapping):
+ raise ValueError(f"map_coefficients[{map_key!r}] missing parameters")
+ if (
+ params.get("acentrics_scale") != 2.0
+ or params.get("centrics_pre_scale") != 1.0
+ or params.get("exclude_free_r_reflections") is not False
+ or params.get("fill_missing") is not False
+ or params.get("isotropize") is not True
+ or params.get("sharp") is not False
+ ):
+ raise ValueError(
+ f"map_coefficients[{map_key!r}] parameters violate preregistered "
+ f"contract: {params!r}"
+ )
+ records = mdata.get("records")
+ if not isinstance(records, list) or len(records) != len(final_by_hkl):
+ raise ValueError(
+ f"map_coefficients[{map_key!r}] record count must equal "
+ f"final_effective count ({len(final_by_hkl)})"
+ )
+ reported_n_refl = _verify_int_count(
+ mdata.get("n_reflections"), f"map_coefficients[{map_key!r}].n_reflections"
+ )
+ reported_centric_count = _verify_nonnegative_int_count(
+ mdata.get("centric_count"), f"map_coefficients[{map_key!r}].centric_count"
+ )
+ reported_acentric_count = _verify_nonnegative_int_count(
+ mdata.get("acentric_count"),
+ f"map_coefficients[{map_key!r}].acentric_count",
+ )
+ if reported_n_refl != len(records):
+ raise ValueError(
+ f"map_coefficients[{map_key!r}].n_reflections {reported_n_refl} != "
+ f"len(records) {len(records)}"
+ )
+ by_hkl: dict[str, Mapping[str, Any]] = {}
+ centric_count = 0
+ for rec in records:
+ rkey = _hkl_key(rec.get("hkl"))
+ if rkey in by_hkl:
+ raise ValueError(
+ f"Duplicate HKL {rkey} in map_coefficients[{map_key!r}]"
+ )
+ if rkey not in final_by_hkl:
+ raise ValueError(
+ f"Unexpected HKL {rkey} in map_coefficients[{map_key!r}]"
+ )
+ re_val = _verify_finite(rec.get("real"), f"{map_key}[{rkey}].real")
+ im_val = _verify_finite(rec.get("imag"), f"{map_key}[{rkey}].imag")
+ amp_val = _verify_finite_nonnegative(
+ rec.get("amplitude"), f"{map_key}[{rkey}].amplitude"
+ )
+ phase_val = _verify_finite(
+ rec.get("phase_deg"), f"{map_key}[{rkey}].phase_deg"
+ )
+ if type(rec.get("is_centric")) is not bool:
+ raise ValueError(f"{map_key}[{rkey}].is_centric must be bool")
+ calc_amp = math.hypot(re_val, im_val)
+ if not math.isclose(calc_amp, amp_val, rel_tol=1e-12, abs_tol=1e-12):
+ raise ValueError(
+ f"{map_key}[{rkey}] amplitude {amp_val} != hypot(real, imag) {calc_amp}"
+ )
+ if calc_amp > 1e-12:
+ calc_phase = math.degrees(math.atan2(im_val, re_val))
+ if _circular_phase_diff_deg(calc_phase, phase_val) > 1e-9:
+ raise ValueError(
+ f"{map_key}[{rkey}] phase_deg {phase_val} disagrees with "
+ f"atan2(imag, real) {calc_phase}"
+ )
+ if rec["is_centric"]:
+ centric_count += 1
+ by_hkl[rkey] = rec
+ acentric_count = len(records) - centric_count
+ if (
+ reported_centric_count != centric_count
+ or reported_acentric_count != acentric_count
+ ):
+ raise ValueError(
+ f"map_coefficients[{map_key!r}] centric/acentric counts "
+ f"({reported_centric_count}, {reported_acentric_count}) != "
+ f"({centric_count}, {acentric_count})"
+ )
+ map_records_by_hkl[map_key] = by_hkl
+ map_summary[map_key] = {
+ "amplitude_label": amp_label,
+ "phase_label": phase_label,
+ "n_reflections": len(records),
+ "centric_count": centric_count,
+ "acentric_count": acentric_count,
+ "parameters": dict(params),
+ "coefficient_semantics": mdata.get("coefficient_semantics"),
+ }
+
+ mtz_supplements = map_coeffs.get("mtz_supplements")
+ if not isinstance(mtz_supplements, Mapping):
+ raise ValueError("map_coefficients missing mtz_supplements")
+ expected_maps_columns = [
+ "H",
+ "K",
+ "L",
+ "2FOFCWT",
+ "PH2FOFCWT",
+ "FOFCWT",
+ "PHFOFCWT",
+ ]
+ expected_obs_pair_columns = [
+ "H",
+ "K",
+ "L",
+ "2FOFCWT",
+ "PH2FOFCWT",
+ ]
+ if mtz_supplements.get("phx_maps_columns") != expected_maps_columns:
+ raise ValueError(
+ f"Unexpected phx_maps.mtz columns: {mtz_supplements.get('phx_maps_columns')!r}"
+ )
+ if mtz_supplements.get("obs_pair_only_columns") != expected_obs_pair_columns:
+ raise ValueError(
+ "Unexpected PHX_OBS_PAIR_ONLY.mtz columns: "
+ f"{mtz_supplements.get('obs_pair_only_columns')!r}"
+ )
+
+ if phx_maps_mtz_path is not None:
+ _sha256_regular_file(Path(phx_maps_mtz_path))
+ loaded_maps_bytes = Path(phx_maps_mtz_path).read_bytes()
+ if phx_maps_mtz_bytes is not None and phx_maps_mtz_bytes != loaded_maps_bytes:
+ raise ValueError("phx_maps_mtz_path bytes disagree with phx_maps_mtz_bytes")
+ maps_bytes = loaded_maps_bytes
+ elif phx_maps_mtz_bytes is not None:
+ maps_bytes = bytes(phx_maps_mtz_bytes)
+ else:
+ raise ValueError(
+ "reconcile_phenix_rfactor_output requires phx_maps_mtz_path or phx_maps_mtz_bytes"
+ )
+
+ if phx_obs_pair_only_mtz_path is not None:
+ _sha256_regular_file(Path(phx_obs_pair_only_mtz_path))
+ loaded_pair_bytes = Path(phx_obs_pair_only_mtz_path).read_bytes()
+ if (
+ phx_obs_pair_only_mtz_bytes is not None
+ and phx_obs_pair_only_mtz_bytes != loaded_pair_bytes
+ ):
+ raise ValueError(
+ "phx_obs_pair_only_mtz_path bytes disagree with phx_obs_pair_only_mtz_bytes"
+ )
+ obs_pair_bytes = loaded_pair_bytes
+ elif phx_obs_pair_only_mtz_bytes is not None:
+ obs_pair_bytes = bytes(phx_obs_pair_only_mtz_bytes)
+ else:
+ raise ValueError(
+ "reconcile_phenix_rfactor_output requires phx_obs_pair_only_mtz_path or "
+ "phx_obs_pair_only_mtz_bytes"
+ )
+
+ maps_sha = _sha256_bytes(maps_bytes)
+ obs_pair_sha = _sha256_bytes(obs_pair_bytes)
+ if mtz_supplements.get("phx_maps_sha256") != maps_sha:
+ raise ValueError(
+ f"phx_maps.mtz SHA-256 {maps_sha} disagrees with "
+ f"mtz_supplements.phx_maps_sha256 {mtz_supplements.get('phx_maps_sha256')!r}"
+ )
+ if mtz_supplements.get("obs_pair_only_sha256") != obs_pair_sha:
+ raise ValueError(
+ f"PHX_OBS_PAIR_ONLY.mtz SHA-256 {obs_pair_sha} disagrees with "
+ f"mtz_supplements.obs_pair_only_sha256 {mtz_supplements.get('obs_pair_only_sha256')!r}"
+ )
+ if cache_manifest_artifacts is not None:
+ if cache_manifest_artifacts.get("phx_maps_mtz", {}).get("sha256") != maps_sha:
+ raise ValueError(
+ "phx_maps.mtz SHA-256 disagrees with cache manifest artifacts"
+ )
+ if (
+ cache_manifest_artifacts.get("phx_obs_pair_only_mtz", {}).get("sha256")
+ != obs_pair_sha
+ ):
+ raise ValueError(
+ "PHX_OBS_PAIR_ONLY.mtz SHA-256 disagrees with cache manifest artifacts"
+ )
+
+ maps_mtz = inspect_map_mtz_bytes(
+ maps_bytes,
+ expected_columns=expected_maps_columns,
+ expected_column_types=["H", "H", "H", "F", "P", "F", "P"],
+ )
+ obs_pair_mtz = inspect_map_mtz_bytes(
+ obs_pair_bytes,
+ expected_columns=expected_obs_pair_columns,
+ expected_column_types=["H", "H", "H", "F", "P"],
+ )
+ if maps_mtz["crystallographic_context"] != obs_pair_mtz["crystallographic_context"]:
+ raise ValueError(
+ "PHX_OBS_PAIR_ONLY.mtz crystallographic context differs from phx_maps.mtz"
+ )
+ maps_ctx = maps_mtz["crystallographic_context"]
+ input_cell = list(validated_input_cell)
+ maps_cell = list(
+ validate_unit_cell_parameters(
+ maps_ctx.get("cell_a_b_c_alpha_beta_gamma"),
+ label="map_mtz.crystallographic_context.cell_a_b_c_alpha_beta_gamma",
+ )
+ )
+ if len(maps_cell) != 6 or any(
+ not math.isclose(a, b, rel_tol=0.0, abs_tol=1e-3)
+ for a, b in zip(input_cell, maps_cell)
+ ):
+ raise ValueError(
+ f"Map MTZ cell {maps_cell} disagrees with input MTZ cell {input_cell}"
+ )
+ input_symm = input_context["symmetry"]
+ maps_symm = maps_ctx["symmetry"]
+ for symm_key in (
+ "operator_count",
+ "primitive_operator_count",
+ "lattice",
+ "space_group_number",
+ "space_group_symbol",
+ "operators_xyz",
+ ):
+ if maps_symm.get(symm_key) != input_symm.get(symm_key):
+ raise ValueError(
+ f"Map MTZ symmetry.{symm_key} {maps_symm.get(symm_key)!r} != "
+ f"input MTZ {input_symm.get(symm_key)!r}"
+ )
+ validated_runtime_space_group = validate_runtime_space_group(
+ crystal_symmetry,
+ input_symm,
+ maps_symmetry=maps_symm,
+ )
+
+ if set(maps_mtz["rows_by_hkl"]) != set(final_by_hkl) or set(
+ obs_pair_mtz["rows_by_hkl"]
+ ) != set(final_by_hkl):
+ raise ValueError(
+ "Map MTZ HKL sets do not match final_effective HKL set"
+ )
+
+ for rkey in final_by_hkl:
+ rec_2fofc = map_records_by_hkl["2mFo-DFc"][rkey]
+ rec_fofc = map_records_by_hkl["mFo-DFc"][rkey]
+ m_row = maps_mtz["rows_by_hkl"][rkey]
+ p_row = obs_pair_mtz["rows_by_hkl"][rkey]
+ if (
+ p_row["hkl"] != m_row["hkl"]
+ or p_row["words"]["2FOFCWT"] != m_row["words"]["2FOFCWT"]
+ or p_row["words"]["PH2FOFCWT"] != m_row["words"]["PH2FOFCWT"]
+ ):
+ raise ValueError(
+ f"PHX_OBS_PAIR_ONLY.mtz 2FOFCWT/PH2FOFCWT words differ from "
+ f"phx_maps.mtz at HKL {rkey}"
+ )
+ amp_2fofc = float(rec_2fofc["amplitude"])
+ ph_2fofc = float(rec_2fofc["phase_deg"])
+ amp_fofc = float(rec_fofc["amplitude"])
+ ph_fofc = float(rec_fofc["phase_deg"])
+ if not math.isclose(
+ m_row["values"]["2FOFCWT"], amp_2fofc, rel_tol=1e-5, abs_tol=1e-5
+ ):
+ raise ValueError(
+ f"phx_maps.mtz 2FOFCWT {m_row['values']['2FOFCWT']} disagrees with "
+ f"JSON amplitude {amp_2fofc} at HKL {rkey}"
+ )
+ if amp_2fofc > 1e-6 and (
+ _circular_phase_diff_deg(m_row["values"]["PH2FOFCWT"], ph_2fofc) > 5e-4
+ ):
+ raise ValueError(
+ f"phx_maps.mtz PH2FOFCWT {m_row['values']['PH2FOFCWT']} disagrees with "
+ f"JSON phase_deg {ph_2fofc} at HKL {rkey}"
+ )
+ if not math.isclose(
+ m_row["values"]["FOFCWT"], amp_fofc, rel_tol=1e-5, abs_tol=1e-5
+ ):
+ raise ValueError(
+ f"phx_maps.mtz FOFCWT {m_row['values']['FOFCWT']} disagrees with "
+ f"JSON amplitude {amp_fofc} at HKL {rkey}"
+ )
+ if amp_fofc > 1e-6 and (
+ _circular_phase_diff_deg(m_row["values"]["PHFOFCWT"], ph_fofc) > 5e-4
+ ):
+ raise ValueError(
+ f"phx_maps.mtz PHFOFCWT {m_row['values']['PHFOFCWT']} disagrees with "
+ f"JSON phase_deg {ph_fofc} at HKL {rkey}"
+ )
+
+ # 7. Compute exact stage-by-stage set differences without inventing per-reflection causes.
+ raw_hkl_set = set(raw_by_hkl)
+ eligible_hkl_set = set(eligible_by_hkl)
+ extracted_hkl_set = set(extracted_by_hkl)
+ final_hkl_set = set(final_by_hkl)
+
+ raw_to_eligible_excluded = sorted(
+ (list(raw_by_hkl[k]["hkl"]) for k in (raw_hkl_set - eligible_hkl_set)),
+ key=tuple,
+ )
+ eligible_to_extracted_excluded = sorted(
+ (list(eligible_by_hkl[k]["hkl"]) for k in (eligible_hkl_set - extracted_hkl_set)),
+ key=tuple,
+ )
+ extracted_to_final_excluded = sorted(
+ (list(extracted_by_hkl[k]["hkl"]) for k in (extracted_hkl_set - final_hkl_set)),
+ key=tuple,
+ )
+
+ return {
+ "reconciliation_status": "verified",
+ "counts": {
+ "raw_mtz_total": len(raw_hkl_set),
+ "input_eligible_total": len(eligible_hkl_set),
+ "extracted_pre_fmodel_total": len(extracted_hkl_set),
+ "final_effective_total": len(final_hkl_set),
+ "actual_work_total": len(work_by_hkl),
+ "actual_free_total": len(free_by_hkl),
+ "excluded_raw_to_input_eligible": len(raw_to_eligible_excluded),
+ "excluded_input_eligible_to_extracted": len(eligible_to_extracted_excluded),
+ "excluded_extracted_to_final_effective": len(extracted_to_final_excluded),
+ },
+ "set_differences": {
+ "raw_to_input_eligible_excluded_hkls": raw_to_eligible_excluded,
+ "input_eligible_to_extracted_excluded_hkls": eligible_to_extracted_excluded,
+ "extracted_to_final_effective_excluded_hkls": extracted_to_final_excluded,
+ },
+ "rfactors": {
+ "r_work": reported_r_work,
+ "r_free": reported_r_free,
+ "r_all": reported_r_all,
+ "r_free_r_work_gap": reported_gap,
+ "replayed_r_work": replayed_r_work,
+ "replayed_r_free": replayed_r_free,
+ "replayed_r_all": replayed_r_all,
+ "replayed_r_free_r_work_gap": replayed_gap,
+ "r_work_abs_diff": abs(replayed_r_work - reported_r_work),
+ "r_free_abs_diff": abs(replayed_r_free - reported_r_free),
+ "r_all_abs_diff": abs(replayed_r_all - reported_r_all),
+ "work_numerator": work_num,
+ "work_denominator": work_den,
+ "free_numerator": free_num,
+ "free_denominator": free_den,
+ "all_numerator": all_num,
+ "all_denominator": all_den,
+ "scale_k1_w": scale_k1_w,
+ "scale_k1_t": scale_k1_t,
+ "scale_k1": scale_k1,
+ "max_abs_f_obs_rescaling_diff": max_f_obs_scale_diff,
+ },
+ "resolution_ranges": reconciled_resolution_ranges,
+ "crystal_symmetry": {
+ "input_header_cell_tokens": list(
+ input_context["cell_a_b_c_alpha_beta_gamma"]
+ ),
+ "input_header_cell_float64": list(validated_input_cell),
+ "expected_ccp4_mtz_float32_cell": list(expected_runtime_float32_cell),
+ "effective_runtime_unit_cell": list(validated_runtime_cell),
+ "max_abs_runtime_vs_float32_diff": max(
+ abs(rt - exp)
+ for rt, exp in zip(
+ validated_runtime_cell, expected_runtime_float32_cell
+ )
+ ),
+ "max_abs_runtime_vs_decimal_header_diff": max(
+ abs(rt - dec)
+ for rt, dec in zip(validated_runtime_cell, validated_input_cell)
+ ),
+ "space_group_symbol": validated_runtime_space_group[
+ "space_group_symbol"
+ ],
+ "space_group_number": validated_runtime_space_group[
+ "space_group_number"
+ ],
+ "raw_space_group_operators_xyz": validated_runtime_space_group[
+ "raw_space_group_operators_xyz"
+ ],
+ "normalized_space_group_operators_xyz": validated_runtime_space_group[
+ "normalized_space_group_operators_xyz"
+ ],
+ "normalized_lookup_symbol": validated_runtime_space_group[
+ "normalized_lookup_symbol"
+ ],
+ "symmetry_normalization_policy": validated_runtime_space_group[
+ "symmetry_normalization_policy"
+ ],
+ "precision_provenance": (
+ "ccp4io/libccp4/ccp4/mtzdata.h:86 and cmtzlib.c:476,549-551 store "
+ "MTZ CELL/DCELL header parameters as IEEE-754 float32 and "
+ "iotbx/mtz/crystal.cpp:61-67 promotes them to float64; strict "
+ "1e-12 stage resolution replay uses the validated effective "
+ "runtime unit cell."
+ ),
+ },
+ "xray_structure_invariance": dict(xrs_check),
+ "map_coefficients_summary": map_summary,
+ "mtz_supplements": dict(mtz_supplements),
+ }
+
+
+def _verify_no_protstruct_overrides() -> None:
+ overrides = sorted(
+ name for name in os.environ if name.startswith("PROTSTRUCT_")
+ )
+ if overrides:
+ raise ValueError(
+ "Canary measurement requires default repository toolchain resolution "
+ f"with no PROTSTRUCT_* environment overrides; found {overrides}"
+ )
+
+
+def run_phenix_rfactor_operation(
+ evidence_root: Path,
+ subject_id: str = "R4",
+ *,
+ repo_root: Path = REPO_ROOT,
+ archive_path: Path | None = None,
+ timeout: float = 900.0,
+ require_no_protstruct_overrides: bool = True,
+ runner: Callable[..., Any] | None = None,
+) -> dict[str, Any]:
+ """Stage ``subject_id`` inputs and run the cached PHENIX first-block R-factor worker.
+
+ Uses :func:`refinement_cache.cached_phenix_operation` via the default
+ toolchain resolver, binds the helper and installed PHENIX Python module
+ sources into the operation identity, verifies that staged inputs remain
+ byte-identical after execution, and reconciles the full-precision output.
+ """
+ if require_no_protstruct_overrides:
+ _verify_no_protstruct_overrides()
+
+ staged = stage_subject_inputs(
+ evidence_root,
+ subject_id=subject_id,
+ repo_root=repo_root,
+ archive_path=archive_path,
+ )
+ fresh_root: Path = staged["evidence_root"]
+ original_pdb_path: Path = staged["original_pdb_path"]
+ original_mtz_path: Path = staged["original_mtz_path"]
+ projected_mtz_path: Path = staged["projected_mtz_path"]
+ input_ledger_path: Path = staged["input_ledger_path"]
+ input_ledger: dict[str, Any] = staged["input_ledger"]
+
+ phenix_dir = fresh_root / "phenix"
+ phenix_dir.mkdir(parents=True, exist_ok=False)
+ preflight_build_json_path = (
+ phenix_dir / "phenix_preflight_build_evidence.json"
+ )
+ admission_failure_json_path = (
+ phenix_dir / "phenix_build_admission_failure.json"
+ )
+ operation_failure_json_path = (
+ phenix_dir / "phenix_operation_failure.json"
+ )
+ params_path = phenix_dir / "phenix_operation_parameters.json"
+ reconciled_json_path = phenix_dir / "phenix_reconciled_manifest.json"
+ cache_dir = phenix_dir / "cache"
+
+ raw_build_evidence = toolchain.phenix_build_evidence()
+ build_info: Any = (
+ dict(raw_build_evidence)
+ if isinstance(raw_build_evidence, Mapping)
+ else raw_build_evidence
+ )
+ onesar_phenix_measure._atomic_write_json_checkpoint(
+ build_info, preflight_build_json_path
+ )
+ preflight_build_sha256 = _sha256_regular_file(preflight_build_json_path)
+
+ preflight_stage = "validate_measured_phenix_build"
+ try:
+ validated_preflight = validate_measured_phenix_build(
+ build_info, expected_version=toolchain.PHENIX_VERSION
+ )
+ expected_resolved_prefix = validated_preflight["resolved_prefix"]
+
+ preflight_stage = "resolve_phenix_vendor_source_files"
+ vendor_sources = resolve_phenix_vendor_source_files(
+ Path(expected_resolved_prefix)
+ )
+ vendor_source_sha256 = {
+ key: _sha256_regular_file(path)
+ for key, path in vendor_sources.items()
+ }
+ except Exception as exc:
+ admission_failure_record = {
+ "schema_version": 1,
+ "status": "rejected",
+ "subject_id": subject_id,
+ "expected_phenix_version": toolchain.PHENIX_VERSION,
+ "preflight_build_evidence_json_path": str(
+ preflight_build_json_path
+ ),
+ "preflight_build_evidence_sha256": preflight_build_sha256,
+ "measured_build": copy.deepcopy(build_info),
+ "failure": {
+ "failed_stage": preflight_stage,
+ "exception_type": type(exc).__name__,
+ "exception_message": str(exc),
+ },
+ "worker_launched": False,
+ }
+ onesar_phenix_measure._atomic_write_json_checkpoint(
+ admission_failure_record, admission_failure_json_path
+ )
+ raise
+
+ helper_script = (_SCRIPTS_DIR / "onesar_phenix_measure.py").resolve(strict=True)
+ _sha256_regular_file(helper_script)
+
+ operation_parameters = {
+ "schema_version": 1,
+ "algorithm": "mmtbx.model_vs_data.first_block_public_api",
+ "subject_id": subject_id,
+ "expected_resolved_prefix": expected_resolved_prefix,
+ "expected_phenix_version": toolchain.PHENIX_VERSION,
+ "labels": list(OBSERVATION_LABELS[:2]),
+ "r_free_flags_label": OBSERVATION_LABELS[2],
+ "test_flag_value": FREE_FLAG_VALUE,
+ "r_free_flags_generate": False,
+ "ignore_r_free_flags": False,
+ "keep_going": False,
+ "scattering_table": "n_gaussian",
+ "twin_law": None,
+ "update_f_part1": True,
+ "apply_pdb_header_resolution_cut": False,
+ "map_coefficients": {
+ "acentrics_scale": 2.0,
+ "centrics_pre_scale": 1.0,
+ "exclude_free_r_reflections": False,
+ "fill_missing": False,
+ "isotropize": True,
+ "sharp": False,
+ "phases_prefix": "PH",
+ },
+ }
+ onesar_phenix_measure.validate_parameters_contract(operation_parameters)
+ onesar_phenix_measure._atomic_write_json_checkpoint(
+ operation_parameters, params_path
+ )
+
+ input_files: dict[str, Path] = {
+ "model_pdb": original_pdb_path,
+ "projected_observations_mtz": projected_mtz_path,
+ "helper_script": helper_script,
+ "parameters_json": params_path,
+ **vendor_sources,
+ }
+ arguments: list[str | Path | refinement_cache.OutputTemplate] = [
+ helper_script,
+ "--pdb",
+ original_pdb_path,
+ "--mtz",
+ projected_mtz_path,
+ "--params-json",
+ params_path,
+ "--out-json",
+ refinement_cache.OutputTemplate("{operation_dir}/phenix_rfactors.json"),
+ "--out-maps-mtz",
+ refinement_cache.OutputTemplate("{operation_dir}/phx_maps.mtz"),
+ "--out-obs-pair-mtz",
+ refinement_cache.OutputTemplate("{operation_dir}/PHX_OBS_PAIR_ONLY.mtz"),
+ "--out-diagnostics-json",
+ refinement_cache.OutputTemplate("{operation_dir}/phenix_diagnostics.json"),
+ "--out-partial-json",
+ refinement_cache.OutputTemplate(
+ "{operation_dir}/phenix_rfactors_partial_evidence.json"
+ ),
+ ]
+ required_outputs: dict[str, str] = {
+ "phenix_rfactors_json": "phenix_rfactors.json",
+ "phx_maps_mtz": "phx_maps.mtz",
+ "phx_obs_pair_only_mtz": "PHX_OBS_PAIR_ONLY.mtz",
+ "phenix_diagnostics_json": "phenix_diagnostics.json",
+ }
+ operation_name = f"onesar_phenix_first_block_rfactors_{subject_id}"
+
+ def _retain_parent_operation_failure(
+ attempt_dir: Path | None,
+ exc: Exception,
+ *,
+ failed_stage: str,
+ ) -> None:
+ retained_artifacts: dict[str, dict[str, str]] = {}
+ diagnostics_summary: dict[str, Any] | None = None
+ partial_summary: dict[str, Any] | None = None
+ if attempt_dir is not None and attempt_dir.is_dir():
+ candidate_files = {
+ "stdout_log": attempt_dir / "stdout.log",
+ "invocation_json": attempt_dir / "invocation.json",
+ "phenix_diagnostics_json": (
+ attempt_dir / "phenix_diagnostics.json"
+ ),
+ "phenix_partial_evidence_json": (
+ attempt_dir / "phenix_rfactors_partial_evidence.json"
+ ),
+ "phx_maps_mtz": attempt_dir / "phx_maps.mtz",
+ "phx_obs_pair_only_mtz": attempt_dir / "PHX_OBS_PAIR_ONLY.mtz",
+ "phenix_rfactors_json": attempt_dir / "phenix_rfactors.json",
+ "manifest_json": attempt_dir / "success.json",
+ }
+ for label, cand_path in candidate_files.items():
+ if cand_path.is_file() and not cand_path.is_symlink():
+ retained_artifacts[label] = {
+ "path": str(cand_path),
+ "sha256": _sha256_regular_file(cand_path),
+ }
+ diag_cand = candidate_files["phenix_diagnostics_json"]
+ if "phenix_diagnostics_json" in retained_artifacts:
+ try:
+ diag_doc = json.loads(
+ diag_cand.read_text(encoding="utf-8")
+ )
+ if isinstance(diag_doc, Mapping):
+ diagnostics_summary = {
+ "status": diag_doc.get("status"),
+ "current_phase": diag_doc.get("current_phase"),
+ "completed_raw_stages": diag_doc.get(
+ "completed_raw_stages"
+ ),
+ "partial_evidence_status": diag_doc.get(
+ "partial_evidence_status"
+ ),
+ "failure": diag_doc.get("failure"),
+ }
+ except Exception:
+ diagnostics_summary = None
+ partial_cand = candidate_files["phenix_partial_evidence_json"]
+ if "phenix_partial_evidence_json" in retained_artifacts:
+ try:
+ partial_doc = json.loads(
+ partial_cand.read_text(encoding="utf-8")
+ )
+ if isinstance(partial_doc, Mapping):
+ partial_summary = {
+ "artifact_role": partial_doc.get("artifact_role"),
+ "status": partial_doc.get("status"),
+ "current_phase": partial_doc.get("current_phase"),
+ "completed_stages": partial_doc.get(
+ "completed_stages"
+ ),
+ "rfactor_replay_verified": partial_doc.get(
+ "rfactor_replay_verified"
+ ),
+ "map_export_verified": partial_doc.get(
+ "map_export_verified"
+ ),
+ "failure": partial_doc.get("failure"),
+ }
+ except Exception:
+ partial_summary = None
+
+ failure_manifest = {
+ "schema_version": 1,
+ "status": "failed",
+ "reusable_cache_output": False,
+ "canonical_eval_qds_eligible": False,
+ "subject_id": subject_id,
+ "failed_stage": failed_stage,
+ "attempt_directory": (
+ str(attempt_dir) if attempt_dir is not None else None
+ ),
+ "returncode": getattr(exc, "returncode", None),
+ "exception_type": type(exc).__name__,
+ "exception_message": str(exc),
+ "preflight_build_evidence_json_path": str(
+ preflight_build_json_path
+ ),
+ "preflight_build_evidence_sha256": preflight_build_sha256,
+ "retained_attempt_artifacts": retained_artifacts,
+ "diagnostics_summary": diagnostics_summary,
+ "partial_evidence_summary": partial_summary,
+ }
+ onesar_phenix_measure._atomic_write_json_checkpoint(
+ failure_manifest, operation_failure_json_path
+ )
+
+ try:
+ try:
+ artifact_paths = refinement_cache.cached_phenix_operation(
+ cache_dir,
+ operation_name,
+ executable="phenix.python",
+ inputs=input_files,
+ arguments=arguments,
+ required_outputs=required_outputs,
+ timeout=timeout,
+ runner=runner,
+ )
+ except Exception as exc:
+ log_p = getattr(exc, "log_path", None)
+ attempt_p = log_p.parent if isinstance(log_p, Path) else None
+ _retain_parent_operation_failure(
+ attempt_p, exc, failed_stage="cached_phenix_operation"
+ )
+ raise
+ finally:
+ # Verify staged inputs were never mutated during execution.
+ if (
+ _sha256_regular_file(original_pdb_path)
+ != staged["admission_record"]["staged_files"]["original_model_pdb_sha256"]
+ ):
+ raise ValueError("Staged original_model.pdb mutated during PHENIX operation")
+ if (
+ _sha256_regular_file(original_mtz_path)
+ != staged["admission_record"]["staged_files"]["original_data_mtz_sha256"]
+ ):
+ raise ValueError("Staged original_data.mtz mutated during PHENIX operation")
+ if (
+ _sha256_regular_file(projected_mtz_path)
+ != staged["admission_record"]["staged_files"][
+ "projected_observations_mtz_sha256"
+ ]
+ ):
+ raise ValueError(
+ "Staged projected_observations.mtz mutated during PHENIX operation"
+ )
+ if (
+ _sha256_regular_file(input_ledger_path)
+ != staged["admission_record"]["staged_files"][
+ "input_hkl_ledger_json_sha256"
+ ]
+ ):
+ raise ValueError(
+ "Staged input_hkl_ledger.json mutated during PHENIX operation"
+ )
+
+ manifest_path = artifact_paths["manifest"]
+ output_json_path = artifact_paths["phenix_rfactors_json"]
+ output_maps_mtz_path = artifact_paths["phx_maps_mtz"]
+ output_obs_pair_mtz_path = artifact_paths["phx_obs_pair_only_mtz"]
+ output_diagnostics_json_path = artifact_paths["phenix_diagnostics_json"]
+ stdout_log_path = artifact_paths["stdout"]
+ bundle_partial_json_path = (
+ output_diagnostics_json_path.parent
+ / "phenix_rfactors_partial_evidence.json"
+ )
+
+ try:
+ manifest_record = json.loads(manifest_path.read_text(encoding="utf-8"))
+ if (
+ not isinstance(manifest_record, Mapping)
+ or type(manifest_record.get("returncode")) is not int
+ or manifest_record["returncode"] != 0
+ or manifest_record.get("reusable") is not True
+ or not output_json_path.is_file()
+ ):
+ raise RuntimeError(
+ f"PHENIX R-factor operation manifest reported invalid/non-reusable status: "
+ f"returncode={manifest_record.get('returncode')!r}, "
+ f"reusable={manifest_record.get('reusable')!r} at {manifest_path}"
+ )
+
+ invocation = manifest_record.get("invocation")
+ if not isinstance(invocation, Mapping):
+ raise ValueError("Cache manifest missing invocation record")
+ validated_manifest_build = validate_measured_phenix_build(
+ invocation.get("measured_build", {}),
+ expected_version=toolchain.PHENIX_VERSION,
+ )
+ if (
+ validated_manifest_build["build_info"]
+ != validated_preflight["build_info"]
+ or validated_manifest_build["resolved_prefix"]
+ != expected_resolved_prefix
+ ):
+ raise ValueError(
+ "Cache manifest measured_build drifted from preflight PHENIX build evidence"
+ )
+ launcher_info = invocation.get("launcher")
+ if (
+ not isinstance(launcher_info, Mapping)
+ or not isinstance(launcher_info.get("path"), str)
+ or not Path(launcher_info["path"]).is_relative_to(
+ Path(expected_resolved_prefix)
+ )
+ ):
+ raise ValueError(
+ f"Cache manifest launcher {launcher_info!r} is not inside "
+ f"{expected_resolved_prefix}"
+ )
+ invocation_inputs = invocation.get("inputs")
+ if not isinstance(invocation_inputs, Mapping):
+ raise ValueError("Cache manifest invocation missing inputs")
+ for mod_key, mod_path in vendor_sources.items():
+ if invocation_inputs.get(mod_key) != {
+ "path": str(mod_path),
+ "sha256": vendor_source_sha256[mod_key],
+ }:
+ raise ValueError(
+ f"Cache manifest input {mod_key} does not match preflight vendor source"
+ )
+
+ manifest_artifacts = manifest_record.get("artifacts")
+ if not isinstance(manifest_artifacts, Mapping):
+ raise ValueError("Cache manifest missing artifacts mapping")
+ if (
+ _sha256_regular_file(output_json_path)
+ != manifest_artifacts.get("phenix_rfactors_json", {}).get("sha256")
+ ):
+ raise ValueError(
+ "phenix_rfactors.json SHA-256 disagrees with cache manifest"
+ )
+
+ phenix_payload = json.loads(
+ output_json_path.read_text(encoding="utf-8")
+ )
+ reconciliation = reconcile_phenix_rfactor_output(
+ phenix_payload,
+ input_ledger,
+ expected_vendor_source_sha256=vendor_source_sha256,
+ expected_vendor_module_paths=vendor_sources,
+ expected_resolved_prefix=expected_resolved_prefix,
+ expected_phenix_version=toolchain.PHENIX_VERSION,
+ measured_build=invocation["measured_build"],
+ phx_maps_mtz_path=output_maps_mtz_path,
+ phx_obs_pair_only_mtz_path=output_obs_pair_mtz_path,
+ phenix_diagnostics_path=output_diagnostics_json_path,
+ cache_manifest_artifacts=manifest_artifacts,
+ )
+ except Exception as exc:
+ _retain_parent_operation_failure(
+ output_diagnostics_json_path.parent,
+ exc,
+ failed_stage="reconcile_phenix_rfactor_output",
+ )
+ raise
+
+ reconciled_record: dict[str, Any] = {
+ "schema_version": 1,
+ "subject": staged["admission_record"]["subject"],
+ "mtz": staged["admission_record"]["mtz"],
+ "staged_files": staged["admission_record"]["staged_files"],
+ "input_hkl_ledger_json_path": str(input_ledger_path),
+ "input_hkl_ledger_json_sha256": staged["admission_record"]["staged_files"][
+ "input_hkl_ledger_json_sha256"
+ ],
+ "preflight_build_evidence_json_path": str(preflight_build_json_path),
+ "preflight_build_evidence_sha256": _sha256_regular_file(
+ preflight_build_json_path
+ ),
+ "cache_artifacts": {
+ key: str(path) for key, path in sorted(artifact_paths.items())
+ },
+ "manifest_path": str(manifest_path),
+ "manifest_sha256": _sha256_regular_file(manifest_path),
+ "stdout_log_path": str(stdout_log_path),
+ "phenix_rfactors_json_path": str(output_json_path),
+ "phenix_diagnostics_json_path": str(output_diagnostics_json_path),
+ "phenix_partial_evidence_json_path": (
+ str(bundle_partial_json_path)
+ if bundle_partial_json_path.is_file()
+ else None
+ ),
+ "phenix_partial_evidence_sha256": (
+ _sha256_regular_file(bundle_partial_json_path)
+ if bundle_partial_json_path.is_file()
+ else None
+ ),
+ "phx_maps_mtz_path": str(output_maps_mtz_path),
+ "phx_obs_pair_only_mtz_path": str(output_obs_pair_mtz_path),
+ "operation_manifest": manifest_record,
+ "reconciliation": reconciliation,
+ }
+ onesar_phenix_measure._atomic_write_json_checkpoint(
+ reconciled_record, reconciled_json_path
+ )
+ return reconciled_record
diff --git a/scripts/test_onesar_refmac_log.py b/scripts/test_onesar_refmac_log.py
new file mode 100644
index 0000000..f2ef8fa
--- /dev/null
+++ b/scripts/test_onesar_refmac_log.py
@@ -0,0 +1,1449 @@
+#!/usr/bin/env python3
+"""Hermetic unit tests for scripts/onesar_refmac_log.py.
+
+All fixtures in this test suite are self-contained in-memory strings with
+explicit provenance labels (historical AMPLE REFMAC 5.8.0049 source excerpt
+and synthetic test cases). No installed vendor tools, external files, or
+network access are required.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import unittest
+from pathlib import Path
+
+_SCRIPTS_DIR = Path(__file__).resolve().parent
+if str(_SCRIPTS_DIR) not in sys.path:
+ sys.path.insert(0, str(_SCRIPTS_DIR))
+
+from onesar_refmac_log import parse_refmac_log # noqa: E402
+
+
+# Provenance: Verbatim structural excerpt from the historical AMPLE test fixture
+# (`ccp4-9/share/ample/testfiles/refmac.log`, SHA-256
+# 15593c60819187660ab7a267ed0232a67bb016262619de06bf56644378df2c2a,
+# CCP4 6.4 / REFMAC 5.8.0049, run date 15/ 7/2014, input 1BYZ-cad.mtz, NCYC 30).
+# Used strictly as parser-shape evidence, never as 1SAR or current-tool evidence.
+HISTORICAL_AMPLE_EXCERPT_LOG = """\
+
+
+ ### CCP4 6.4: Refmac_5.8.0049 version 5.8.0049 : 30/06/13##
+ User: unknown Run date: 15/ 7/2014 Run time: 21:14:02
+ $TEXT:Reference1: $$ Main reference $$
+ "REFMAC5 for the refinement of macromolecular crystal structures:"
+ $$
+ $SUMMARY :Reference1: $$ Refmac: $$
+ :TEXT:Reference1: $$
+ Data line--- LABIN FP=FP SIGFP=SIGFP FREE=FREE
+ Data line--- NCYC 30
+ Data line--- END
+ Refinement cycles : 30
+
+$TABLE: Cycle 1. Rfactor analysis, F distribution v resln :
+$GRAPHS:Cycle 1. v. resln :N:1,6,7,11,12:
+:Cycle 1. and v. resln :N:1,4,5,9,10:
+:Cycle 1. % observed v. resln :N:1,3:
+$$
+M(4SSQ/LL) NR_used %_obs M(Fo_used) M(Fc_used) Rf_used WR_used
+NR_free M(Fo_free) M(Fc_free) Rf_free WR_free $$
+$$
+ 0.032 293 99.36 39.1 31.2 0.54 0.77 19 24.6 36.2 0.76 1.48
+ 0.093 548 100.00 35.1 25.7 0.53 0.61 35 32.9 26.5 0.53 0.59
+ 0.154 675 100.00 28.2 23.8 0.51 0.62 35 23.7 21.4 0.58 0.70
+ 0.215 853 100.00 23.6 18.9 0.55 0.65 39 25.4 18.8 0.54 0.59
+ 0.276 920 100.00 18.0 14.4 0.55 0.60 48 16.2 13.9 0.47 0.55
+ 0.338 1031 100.00 14.7 11.9 0.51 0.54 70 13.6 11.4 0.51 0.53
+ 0.399 1106 100.00 13.4 9.9 0.55 0.55 68 13.1 10.1 0.45 0.45
+ 0.460 1193 100.00 12.3 9.5 0.52 0.54 68 10.4 9.3 0.45 0.47
+ 0.521 1279 99.85 10.5 8.3 0.55 0.55 72 10.8 8.7 0.46 0.48
+ 0.582 1356 99.72 10.0 8.0 0.53 0.52 75 11.8 7.8 0.58 0.58
+ 0.643 1364 99.18 9.8 7.6 0.55 0.55 82 9.2 7.2 0.52 0.53
+ 0.704 1473 98.72 9.1 6.8 0.55 0.54 70 9.7 6.5 0.56 0.54
+ 0.765 1546 97.89 8.9 6.3 0.56 0.56 79 8.0 5.8 0.55 0.53
+ 0.826 1484 92.33 7.9 5.9 0.54 0.53 82 8.0 5.9 0.55 0.52
+ 0.887 1434 87.55 7.5 5.5 0.55 0.53 85 7.6 5.1 0.57 0.54
+ 0.948 1468 82.59 6.5 4.9 0.54 0.52 50 5.7 5.3 0.45 0.43
+ 1.010 1351 75.18 5.4 4.1 0.57 0.54 73 4.7 4.5 0.64 0.60
+ 1.071 1200 67.16 4.4 3.5 0.57 0.56 70 4.2 3.8 0.53 0.50
+ 1.132 1143 58.99 3.9 3.0 0.53 0.53 51 3.5 3.0 0.57 0.60
+ 1.193 750 39.07 3.8 2.6 0.53 0.54 42 3.9 2.5 0.51 0.53
+$$
+
+ $TABLE: Cycle 1. Fom(-acentric, centric, overall v resln:
+ $GRAPHS:Cycle 1. M(Fom) v. resln :N:1,3,5,7,8:
+ $$
+ <4SSQ/LL> NREFa FOMa NREFc FOMc NREFall FOMall SigmaA_Fc1 $$
+ $$
+ 0.0321 293 0.567 0 0.000 293 0.567 0.646
+ 0.0932 548 0.567 0 0.000 548 0.567 0.658
+ 0.1542 675 0.572 0 0.000 675 0.572 0.667
+ 0.2153 853 0.511 0 0.000 853 0.511 0.675
+ 0.2764 920 0.442 0 0.000 920 0.442 0.681
+ 0.3375 1031 0.422 0 0.000 1031 0.422 0.693
+ 0.3986 1106 0.447 0 0.000 1106 0.447 0.729
+ 0.4597 1193 0.483 0 0.000 1193 0.483 0.721
+ 0.5208 1279 0.316 0 0.000 1279 0.316 0.524
+ 0.5819 1356 0.246 0 0.000 1356 0.246 0.430
+ 0.6430 1364 0.282 0 0.000 1364 0.282 0.529
+ 0.7041 1473 0.307 0 0.000 1473 0.307 0.580
+ 0.7652 1546 0.333 0 0.000 1546 0.333 0.574
+ 0.8262 1484 0.270 0 0.000 1484 0.270 0.489
+ 0.8873 1434 0.219 0 0.000 1434 0.219 0.410
+ 0.9484 1468 0.304 0 0.000 1468 0.304 0.537
+ 1.0095 1351 0.409 0 0.000 1351 0.409 0.642
+ 1.0706 1200 0.256 0 0.000 1200 0.256 0.445
+ 1.1317 1143 0.279 0 0.000 1143 0.279 0.478
+ 1.1928 750 0.193 0 0.000 750 0.193 0.361
+ $$
+Resolution limits = 25.690 0.904
+Number of used reflections = 22467
+Percentage observed = 85.8624
+Percentage of free reflections = 5.1225
+Overall R factor = 0.5397
+Free R factor = 0.5317
+Overall weighted R factor = 0.5603
+Free weighted R factor = 0.5489
+Overall weighted R2 factor = 0.7148
+Free weighted R2 factor = 0.7169
+Overall figure of merit = 0.3465
+-----------------------------------------------------------------------------
+Overall R factor = 0.5380
+Free R factor = 0.5313
+Overall figure of merit = 0.3343
+-----------------------------------------------------------------------------
+Overall R factor = 0.5363
+Free R factor = 0.5315
+Overall figure of merit = 0.3314
+-----------------------------------------------------------------------------
+Overall R factor = 0.5352
+Free R factor = 0.5321
+Overall figure of merit = 0.3295
+-----------------------------------------------------------------------------
+Overall R factor = 0.5344
+Free R factor = 0.5327
+Overall figure of merit = 0.3301
+-----------------------------------------------------------------------------
+Overall R factor = 0.5338
+Free R factor = 0.5330
+Overall figure of merit = 0.3294
+-----------------------------------------------------------------------------
+Overall R factor = 0.5334
+Free R factor = 0.5333
+Overall figure of merit = 0.3285
+-----------------------------------------------------------------------------
+Overall R factor = 0.5330
+Free R factor = 0.5336
+Overall figure of merit = 0.3282
+-----------------------------------------------------------------------------
+Overall R factor = 0.5327
+Free R factor = 0.5339
+Overall figure of merit = 0.3273
+-----------------------------------------------------------------------------
+Overall R factor = 0.5324
+Free R factor = 0.5343
+Overall figure of merit = 0.3269
+-----------------------------------------------------------------------------
+Overall R factor = 0.5322
+Free R factor = 0.5344
+Overall figure of merit = 0.3259
+-----------------------------------------------------------------------------
+Overall R factor = 0.5319
+Free R factor = 0.5345
+Overall figure of merit = 0.3246
+-----------------------------------------------------------------------------
+Overall R factor = 0.5315
+Free R factor = 0.5345
+Overall figure of merit = 0.3239
+-----------------------------------------------------------------------------
+Overall R factor = 0.5311
+Free R factor = 0.5345
+Overall figure of merit = 0.3235
+-----------------------------------------------------------------------------
+Overall R factor = 0.5307
+Free R factor = 0.5346
+Overall figure of merit = 0.3237
+-----------------------------------------------------------------------------
+Overall R factor = 0.5302
+Free R factor = 0.5342
+Overall figure of merit = 0.3226
+-----------------------------------------------------------------------------
+Overall R factor = 0.5297
+Free R factor = 0.5338
+Overall figure of merit = 0.3219
+-----------------------------------------------------------------------------
+Overall R factor = 0.5291
+Free R factor = 0.5331
+Overall figure of merit = 0.3213
+-----------------------------------------------------------------------------
+Overall R factor = 0.5283
+Free R factor = 0.5326
+Overall figure of merit = 0.3222
+-----------------------------------------------------------------------------
+Overall R factor = 0.5274
+Free R factor = 0.5321
+Overall figure of merit = 0.3225
+-----------------------------------------------------------------------------
+Overall R factor = 0.5265
+Free R factor = 0.5316
+Overall figure of merit = 0.3227
+-----------------------------------------------------------------------------
+Overall R factor = 0.5258
+Free R factor = 0.5308
+Overall figure of merit = 0.3233
+-----------------------------------------------------------------------------
+Overall R factor = 0.5251
+Free R factor = 0.5301
+Overall figure of merit = 0.3232
+-----------------------------------------------------------------------------
+Overall R factor = 0.5245
+Free R factor = 0.5295
+Overall figure of merit = 0.3256
+-----------------------------------------------------------------------------
+Overall R factor = 0.5239
+Free R factor = 0.5291
+Overall figure of merit = 0.3288
+-----------------------------------------------------------------------------
+Overall R factor = 0.5234
+Free R factor = 0.5287
+Overall figure of merit = 0.3318
+-----------------------------------------------------------------------------
+Overall R factor = 0.5228
+Free R factor = 0.5287
+Overall figure of merit = 0.3347
+-----------------------------------------------------------------------------
+Overall R factor = 0.5222
+Free R factor = 0.5286
+Overall figure of merit = 0.3394
+-----------------------------------------------------------------------------
+Overall R factor = 0.5217
+Free R factor = 0.5286
+Overall figure of merit = 0.3400
+-----------------------------------------------------------------------------
+Overall R factor = 0.5211
+Free R factor = 0.5288
+Overall figure of merit = 0.3401
+-----------------------------------------------------------------------------
+
+$TABLE: Cycle 31. Rfactor analysis, F distribution v resln :
+$GRAPHS:Cycle 31. v. resln :N:1,6,7,11,12:
+:Cycle 31. and v. resln :N:1,4,5,9,10:
+:Cycle 31. % observed v. resln :N:1,3:
+$$
+M(4SSQ/LL) NR_used %_obs M(Fo_used) M(Fc_used) Rf_used WR_used
+NR_free M(Fo_free) M(Fc_free) Rf_free WR_free $$
+$$
+ 0.032 293 99.36 39.1 31.1 0.53 0.77 19 24.6 35.0 0.74 1.36
+ 0.093 548 100.00 35.1 26.5 0.52 0.61 35 32.9 26.5 0.58 0.64
+ 0.154 675 100.00 28.2 25.1 0.49 0.62 35 23.7 23.8 0.60 0.74
+ 0.215 853 100.00 23.6 19.6 0.52 0.62 39 25.4 20.0 0.50 0.58
+ 0.276 920 100.00 18.0 14.2 0.52 0.56 48 16.2 13.1 0.49 0.56
+ 0.338 1031 100.00 14.7 11.4 0.51 0.54 70 13.6 11.0 0.50 0.51
+ 0.399 1106 100.00 13.4 10.2 0.53 0.54 68 13.1 10.7 0.44 0.46
+ 0.460 1193 100.00 12.3 10.0 0.50 0.51 68 10.4 9.2 0.42 0.44
+ 0.521 1279 99.85 10.5 8.7 0.51 0.52 72 10.8 9.7 0.47 0.50
+ 0.582 1356 99.72 10.0 8.4 0.51 0.50 75 11.8 8.3 0.52 0.50
+ 0.643 1364 99.18 9.8 7.5 0.53 0.52 82 9.2 7.5 0.53 0.50
+ 0.704 1473 98.72 9.1 6.8 0.53 0.52 70 9.7 6.5 0.55 0.53
+ 0.765 1546 97.89 8.8 6.5 0.55 0.55 79 8.0 6.2 0.53 0.51
+ 0.826 1484 92.33 7.9 6.2 0.53 0.52 82 8.0 5.7 0.53 0.52
+ 0.887 1434 87.55 7.5 5.7 0.51 0.49 85 7.6 5.3 0.52 0.50
+ 0.948 1468 82.59 6.5 4.9 0.52 0.50 50 5.7 4.9 0.60 0.56
+ 1.010 1351 75.18 5.4 4.2 0.56 0.54 73 4.7 4.4 0.65 0.60
+ 1.071 1200 67.16 4.4 3.7 0.56 0.55 70 4.2 3.4 0.53 0.51
+ 1.132 1143 58.99 3.9 3.1 0.55 0.55 51 3.5 3.6 0.61 0.57
+ 1.193 750 39.07 3.8 2.7 0.53 0.53 42 3.9 2.9 0.55 0.58
+$$
+
+ $TABLE: Cycle 31. Fom(-acentric, centric, overall v resln:
+ $GRAPHS:Cycle 31. M(Fom) v. resln :N:1,3,5,7,8:
+ $$
+ <4SSQ/LL> NREFa FOMa NREFc FOMc NREFall FOMall SigmaA_Fc1 $$
+ $$
+ 0.0321 293 0.581 0 0.000 293 0.581 0.654
+ 0.0932 548 0.596 0 0.000 548 0.596 0.672
+ 0.1542 675 0.611 0 0.000 675 0.611 0.685
+ 0.2153 853 0.540 0 0.000 853 0.540 0.696
+ 0.2764 920 0.456 0 0.000 920 0.456 0.705
+ 0.3375 1031 0.421 0 0.000 1031 0.421 0.718
+ 0.3986 1106 0.471 0 0.000 1106 0.471 0.751
+ 0.4597 1193 0.531 0 0.000 1193 0.531 0.740
+ 0.5208 1279 0.344 0 0.000 1279 0.344 0.531
+ 0.5819 1356 0.222 0 0.000 1356 0.222 0.364
+ 0.6430 1364 0.165 0 0.000 1364 0.165 0.304
+ 0.7041 1473 0.261 0 0.000 1473 0.261 0.470
+ 0.7652 1546 0.370 0 0.000 1546 0.370 0.602
+ 0.8262 1484 0.269 0 0.000 1484 0.269 0.463
+ 0.8873 1434 0.235 0 0.000 1434 0.235 0.409
+ 0.9484 1468 0.282 0 0.000 1468 0.282 0.496
+ 1.0095 1351 0.353 0 0.000 1351 0.353 0.588
+ 1.0706 1200 0.308 0 0.000 1200 0.308 0.501
+ 1.1317 1143 0.225 0 0.000 1143 0.225 0.387
+ 1.1928 750 0.136 0 0.000 750 0.136 0.246
+ $$
+Resolution limits = 25.690 0.904
+Number of used reflections = 22467
+Percentage observed = 85.8624
+Percentage of free reflections = 5.1225
+Overall R factor = 0.5207
+Free R factor = 0.5289
+Overall weighted R factor = 0.5425
+Free weighted R factor = 0.5425
+Overall weighted R2 factor = 0.6996
+Free weighted R2 factor = 0.6953
+Overall correlation coefficient = 0.6076
+Free correlation coefficient = 0.5684
+Cruickshanks DPI for coordinate error= 0.0241
+DPI based on free R factor = 0.0244
+Overall figure of merit = 0.3407
+ML based su of positional parameters = 0.0589
+ML based su of thermal parameters = 1.0064
+-----------------------------------------------------------------------------
+
+$TABLE: Rfactor analysis, stats vs cycle :
+$GRAPHS: vs cycle :N:1,2,3:
+:FOM vs cycle :N:1,4:
+:-LL vs cycle :N:1,5:
+:-LLfree vs cycle :N:1,6:
+:Geometry vs cycle:N:1,7,8,9,10,11:
+$$
+ Ncyc Rfact Rfree FOM -LL -LLfree rmsBOND zBOND rmsANGL zANGL rmsCHIRAL $$
+$$
+ 0 0.5397 0.5317 0.347 66009. 3533.2 0.0193 0.987 3.189 1.772 0.092
+ 1 0.5380 0.5313 0.334 66023. 3532.7 0.0108 0.528 2.391 1.304 0.081
+ 2 0.5363 0.5315 0.331 66014. 3532.9 0.0088 0.422 2.037 1.111 0.056
+ 3 0.5352 0.5321 0.329 65921. 3528.3 0.0075 0.359 1.844 1.008 0.036
+ 4 0.5344 0.5327 0.330 65815. 3523.4 0.0064 0.311 1.713 0.940 0.023
+ 5 0.5338 0.5330 0.329 65768. 3521.5 0.0057 0.274 1.614 0.887 0.018
+ 6 0.5334 0.5333 0.329 65708. 3518.9 0.0051 0.245 1.532 0.843 0.019
+ 7 0.5330 0.5336 0.328 65709. 3519.7 0.0046 0.223 1.462 0.805 0.021
+ 8 0.5327 0.5339 0.327 65762. 3523.2 0.0042 0.206 1.400 0.771 0.024
+ 9 0.5324 0.5343 0.327 65720. 3521.4 0.0040 0.192 1.345 0.741 0.026
+ 10 0.5322 0.5344 0.326 65744. 3523.5 0.0037 0.182 1.296 0.713 0.028
+ 11 0.5319 0.5345 0.325 65745. 3524.1 0.0036 0.173 1.250 0.688 0.029
+ 12 0.5315 0.5345 0.324 65724. 3523.4 0.0034 0.166 1.206 0.662 0.029
+ 13 0.5311 0.5345 0.324 65753. 3525.6 0.0033 0.159 1.164 0.639 0.029
+ 14 0.5307 0.5346 0.324 65784. 3527.6 0.0031 0.154 1.126 0.617 0.030
+ 15 0.5302 0.5342 0.323 65865. 3531.8 0.0031 0.151 1.091 0.597 0.030
+ 16 0.5297 0.5338 0.322 65905. 3534.0 0.0030 0.150 1.059 0.579 0.030
+ 17 0.5291 0.5331 0.321 65958. 3536.7 0.0030 0.150 1.028 0.562 0.029
+ 18 0.5283 0.5326 0.322 65955. 3536.5 0.0029 0.149 1.000 0.545 0.029
+ 19 0.5274 0.5321 0.323 65901. 3532.7 0.0029 0.149 0.972 0.529 0.029
+ 20 0.5265 0.5316 0.323 65870. 3530.9 0.0029 0.150 0.946 0.515 0.029
+ 21 0.5258 0.5308 0.323 65814. 3527.7 0.0029 0.151 0.921 0.500 0.028
+ 22 0.5251 0.5301 0.323 65800. 3525.9 0.0029 0.152 0.897 0.487 0.028
+ 23 0.5245 0.5295 0.326 65799. 3525.3 0.0029 0.153 0.874 0.474 0.028
+ 24 0.5239 0.5291 0.329 65806. 3525.1 0.0029 0.154 0.851 0.462 0.027
+ 25 0.5234 0.5287 0.332 65841. 3527.3 0.0029 0.155 0.829 0.450 0.027
+ 26 0.5228 0.5287 0.335 65771. 3522.6 0.0029 0.156 0.809 0.438 0.026
+ 27 0.5222 0.5286 0.339 65628. 3514.3 0.0029 0.156 0.789 0.427 0.026
+ 28 0.5217 0.5286 0.340 65520. 3508.0 0.0029 0.156 0.769 0.417 0.025
+ 29 0.5211 0.5288 0.340 65506. 3506.9 0.0029 0.156 0.751 0.407 0.025
+ 30 0.5207 0.5289 0.341 65565. 3509.9 0.0028 0.155 0.733 0.397 0.024
+ $$
+ $TEXT:Result: $$ Final results $$
+ Initial Final
+ R factor 0.5397 0.5207
+ R free 0.5317 0.5289
+ Rms BondLength 0.0193 0.0028
+ Rms BondAngle 3.1891 0.7330
+ Rms ChirVolume 0.0916 0.0240
+ $$
+"""
+
+
+# Provenance: Purely synthetic single-cycle (NCYC 0) log fixture testing the
+# Cycle 1 / Ncyc 0 correspondence where detailed resolution table Cycle 1
+# corresponds to cycle-summary row Ncyc 0.
+SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG = """\
+ ### CCP4 9.0: Refmac_5.8.0430 version 5.8.0430 : 01/01/24##
+ Data line--- NCYC 0
+ Refinement cycles : 0
+
+$TABLE: Cycle 1. Rfactor analysis, F distribution v resln :
+$GRAPHS:Cycle 1. v. resln :N:1,6,7,11,12:
+$$
+M(4SSQ/LL) NR_used %_obs M(Fo_used) M(Fc_used) Rf_used WR_used
+NR_free M(Fo_free) M(Fc_free) Rf_free WR_free $$
+$$
+ 0.050 3100 99.50 42.0 40.1 0.18 0.21 160 41.2 39.0 0.22 0.25
+ 0.150 3432 98.10 21.5 20.4 0.20 0.23 184 20.8 19.5 0.24 0.27
+$$
+
+ $TABLE: Cycle 1. Fom(-acentric, centric, overall v resln:
+ $GRAPHS:Cycle 1. M(Fom) v. resln :N:1,3,5,7,8:
+ $$
+ <4SSQ/LL> NREFa FOMa NREFc FOMc NREFall FOMall SigmaA_Fc1 $$
+ $$
+ 0.0500 3000 0.850 100 0.780 3100 0.848 0.910
+ 0.1500 3332 0.810 100 0.740 3432 0.808 0.885
+ $$
+Resolution limits = 35.000 1.800
+Number of used reflections = 6532
+Percentage observed = 98.7500
+Percentage of free reflections = 5.0029
+Overall R factor = 0.1912
+Free R factor = 0.2315
+Overall weighted R factor = 0.2210
+Free weighted R factor = 0.2610
+Overall weighted R2 factor = 0.2890
+Free weighted R2 factor = 0.3310
+Overall figure of merit = 0.8270
+-----------------------------------------------------------------------------
+
+$TABLE: Rfactor analysis, stats vs cycle :
+$GRAPHS: vs cycle :N:1,2,3:
+$$
+ Ncyc Rfact Rfree FOM -LL -LLfree rmsBOND zBOND rmsANGL zANGL rmsCHIRAL $$
+$$
+ 0 0.1912 0.2315 0.827 12345. 654.3 0.0081 0.410 1.120 0.550 0.045
+ $$
+ $TEXT:Result: $$ Final results $$
+ Initial Final
+ R factor 0.1912 0.1912
+ R free 0.2315 0.2315
+ Rms BondLength 0.0081 0.0081
+ Rms BondAngle 1.1200 1.1200
+ Rms ChirVolume 0.0450 0.0450
+ $$
+"""
+
+
+class TestOnesarRefmacLogParser(unittest.TestCase):
+ """Hermetic unit tests for parse_refmac_log."""
+
+ def test_historical_ample_excerpt_reconciles_all_cycles_and_bins(self) -> None:
+ parsed = parse_refmac_log(HISTORICAL_AMPLE_EXCERPT_LOG)
+ # Ensure output is completely JSON-serializable
+ roundtrip = json.loads(json.dumps(parsed))
+ self.assertEqual(roundtrip["reconciled_final_cycle"], parsed["reconciled_final_cycle"])
+
+ self.assertEqual(
+ parsed["source_metadata"]["reported_refmac_version"], "5.8.0049"
+ )
+ self.assertEqual(parsed["source_metadata"]["reported_suite"], "CCP4 6.4")
+ self.assertEqual(parsed["source_metadata"]["requested_ncyc"], 30)
+
+ self.assertEqual(
+ [b["id"] for b in parsed["table_blocks"]],
+ [
+ "initial_resolution_rfactor",
+ "initial_resolution_fom",
+ "final_resolution_rfactor",
+ "final_resolution_fom",
+ "cycle_summary",
+ ],
+ )
+
+ # Verify summary and detailed cycles
+ self.assertEqual(parsed["cycle_summary"]["row_count"], 31)
+ self.assertEqual(parsed["cycle_summary"]["summary_cycle_min"], 0)
+ self.assertEqual(parsed["cycle_summary"]["summary_cycle_max"], 30)
+ self.assertEqual(len(parsed["detailed_cycles"]), 31)
+ self.assertTrue(
+ parsed["cycle_r_values"]["all_ordered_detail_pairs_equal_summary_values"]
+ )
+ self.assertTrue(
+ parsed["consistency_checks"][
+ "initial_detail_summary_and_final_results_initial_values_equal"
+ ]
+ )
+ self.assertTrue(
+ parsed["consistency_checks"][
+ "final_detail_summary_and_final_results_final_values_equal"
+ ]
+ )
+ self.assertEqual(
+ parsed["cycle_r_values"]["detail_rfactor_table_cycle_labels"], [1, 31]
+ )
+
+ # Initial and final R pairs and precision
+ init_cyc = parsed["reconciled_initial_cycle"]
+ self.assertEqual(init_cyc["summary_ncyc"], 0)
+ self.assertEqual(init_cyc["detailed_cycle"], 1)
+ self.assertEqual(init_cyc["r_work_token"], "0.5397")
+ self.assertEqual(init_cyc["r_free_token"], "0.5317")
+ self.assertEqual(init_cyc["r_work_decimal_places"], 4)
+ self.assertEqual(init_cyc["r_free_decimal_places"], 4)
+
+ final_cyc = parsed["reconciled_final_cycle"]
+ self.assertEqual(final_cyc["summary_ncyc"], 30)
+ self.assertEqual(final_cyc["detailed_cycle"], 31)
+ self.assertEqual(final_cyc["resolution_table_cycle"], 31)
+ self.assertEqual(final_cyc["r_work_token"], "0.5207")
+ self.assertEqual(final_cyc["r_free_token"], "0.5289")
+ self.assertAlmostEqual(final_cyc["r_work"], 0.5207)
+ self.assertAlmostEqual(final_cyc["r_free"], 0.5289)
+ self.assertEqual(final_cyc["r_work_decimal_places"], 4)
+ self.assertEqual(final_cyc["r_free_decimal_places"], 4)
+ self.assertEqual(final_cyc["nr_used_sum"], 22467)
+ self.assertEqual(final_cyc["nr_free_sum"], 1213)
+ self.assertEqual(final_cyc["combined_reported_count"], 23680)
+
+ # Verify unweighted R is distinct from weighted R, R2, and FOM
+ final_detail = parsed["detailed_cycles"][-1]
+ self.assertEqual(final_detail["overall_r_token"], "0.5207")
+ self.assertEqual(final_detail["free_r_token"], "0.5289")
+ self.assertEqual(final_detail["overall_weighted_r_token"], "0.5425")
+ self.assertEqual(final_detail["free_weighted_r_token"], "0.5425")
+ self.assertEqual(final_detail["overall_weighted_r2_token"], "0.6996")
+ self.assertEqual(final_detail["free_weighted_r2_token"], "0.6953")
+ self.assertEqual(final_detail["overall_fom_token"], "0.3407")
+
+ # Verify per-bin counts and 2-decimal-place R tokens
+ final_bins = parsed["final_resolution_bins"]
+ self.assertEqual(final_bins["bin_count"], 20)
+ self.assertEqual(final_bins["nr_used_sum"], 22467)
+ self.assertEqual(final_bins["nr_free_sum"], 1213)
+ self.assertEqual(final_bins["rf_used_printed_decimal_places"], 2)
+ self.assertEqual(final_bins["rf_free_printed_decimal_places"], 2)
+ self.assertEqual(final_bins["bins"][0]["nr_used"], 293)
+ self.assertEqual(final_bins["bins"][0]["nr_free"], 19)
+ self.assertEqual(final_bins["bins"][0]["rf_used_token"], "0.53")
+ self.assertEqual(final_bins["bins"][0]["rf_free_token"], "0.74")
+ self.assertEqual(final_bins["bins"][0]["wr_free_token"], "1.36")
+ self.assertEqual(final_bins["bins"][-1]["nr_used"], 750)
+ self.assertEqual(final_bins["bins"][-1]["nr_free"], 42)
+ self.assertEqual(final_bins["bins"][-1]["rf_used_token"], "0.53")
+ self.assertEqual(final_bins["bins"][-1]["rf_free_token"], "0.55")
+
+ def test_single_cycle_cycle1_ncyc0_correspondence(self) -> None:
+ parsed = parse_refmac_log(SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG)
+ self.assertEqual(parsed["cycle_summary"]["row_count"], 1)
+ self.assertEqual(parsed["cycle_summary"]["summary_cycle_min"], 0)
+ self.assertEqual(parsed["cycle_summary"]["summary_cycle_max"], 0)
+ self.assertEqual(len(parsed["detailed_cycles"]), 1)
+ self.assertEqual(parsed["detailed_cycles"][0]["detailed_cycle"], 1)
+ self.assertEqual(parsed["detailed_cycles"][0]["summary_ncyc"], 0)
+
+ final_cyc = parsed["reconciled_final_cycle"]
+ self.assertEqual(final_cyc["summary_ncyc"], 0)
+ self.assertEqual(final_cyc["detailed_cycle"], 1)
+ self.assertEqual(final_cyc["resolution_table_cycle"], 1)
+ self.assertEqual(final_cyc["r_work_token"], "0.1912")
+ self.assertEqual(final_cyc["r_free_token"], "0.2315")
+ self.assertEqual(final_cyc["nr_used_sum"], 6532)
+ self.assertEqual(final_cyc["nr_free_sum"], 344)
+ self.assertEqual(final_cyc["combined_reported_count"], 6876)
+
+ def test_unweighted_r_above_one_accepted_and_precision_preserved(self) -> None:
+ log_text = (
+ SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace("0.1912", "1.12500")
+ .replace("0.2315", "1.34000")
+ .replace("0.18 0.21", "1.080 1.210")
+ .replace("0.22 0.25", "1.420 1.550")
+ .replace("0.20 0.23", "1.150 1.280")
+ .replace("0.24 0.27", "1.260 1.390")
+ )
+ parsed = parse_refmac_log(log_text)
+ final_cyc = parsed["reconciled_final_cycle"]
+ self.assertEqual(final_cyc["r_work_token"], "1.12500")
+ self.assertEqual(final_cyc["r_free_token"], "1.34000")
+ self.assertAlmostEqual(final_cyc["r_work"], 1.125)
+ self.assertAlmostEqual(final_cyc["r_free"], 1.34)
+ self.assertEqual(final_cyc["r_work_decimal_places"], 5)
+ self.assertEqual(final_cyc["r_free_decimal_places"], 5)
+ self.assertEqual(
+ parsed["final_resolution_bins"]["rf_used_printed_decimal_places"], 3
+ )
+ self.assertEqual(
+ parsed["final_resolution_bins"]["rf_free_printed_decimal_places"], 3
+ )
+
+ def test_competing_unrelated_tables_ignored_for_rfactor_extraction(self) -> None:
+ competing_before = """\
+$TABLE: Cycle 99. Anisotropic scaling v resln :
+$GRAPHS:Cycle 99. Scale v. resln :N:1,2,3:
+$$
+M(4SSQ/LL) Scale_k B_aniso $$
+$$
+ 0.050 0.9999 0.8888
+ 0.150 0.7777 0.6666
+$$
+"""
+ competing_after = """\
+$TABLE: Restraint geometry stats vs cycle :
+$GRAPHS:Restraint rms vs cycle :N:1,2,3:
+$$
+Ncyc rmsBOND rmsANGL $$
+$$
+ 0 0.9111 0.9222
+$$
+"""
+ augmented = (
+ competing_before
+ + SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG
+ + competing_after
+ )
+ parsed = parse_refmac_log(augmented)
+ self.assertEqual(
+ [b["table_kind"] for b in parsed["table_blocks"]],
+ ["other", "resolution_rfactor", "resolution_fom", "cycle_summary", "other"],
+ )
+ self.assertEqual(parsed["reconciled_final_cycle"]["r_work_token"], "0.1912")
+ self.assertEqual(parsed["reconciled_final_cycle"]["r_free_token"], "0.2315")
+ self.assertEqual(parsed["reconciled_final_cycle"]["nr_used_sum"], 6532)
+ self.assertEqual(parsed["reconciled_final_cycle"]["nr_free_sum"], 344)
+
+ # Log with only competing tables and no Rfactor table must fail
+ with self.assertRaisesRegex(ValueError, "Missing required"):
+ parse_refmac_log(competing_before + competing_after)
+
+ def test_standalone_header_end_double_dollar_accepted(self) -> None:
+ log_text = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "NR_free M(Fo_free) M(Fc_free) Rf_free WR_free $$\n$$",
+ "NR_free M(Fo_free) M(Fc_free) Rf_free WR_free\n$$\n$$",
+ )
+ parsed = parse_refmac_log(log_text)
+ roles = parsed["final_resolution_bins"]["double_dollar_roles"]
+ self.assertEqual(roles["data_start_line"], roles["header_end_line"] + 1)
+ self.assertEqual(parsed["final_resolution_bins"]["nr_used_sum"], 6532)
+
+ def test_malformed_delimiters_rejected(self) -> None:
+ # 1. Missing data_start $$ (only 3 $$ in resolution table)
+ missing_third_dd = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "NR_free M(Fo_free) M(Fc_free) Rf_free WR_free $$\n$$",
+ "NR_free M(Fo_free) M(Fc_free) Rf_free WR_free $$",
+ )
+ with self.assertRaisesRegex(ValueError, "data_start \\$\\$"):
+ parse_refmac_log(missing_third_dd)
+
+ # 2. Missing closing data_end $$ on cycle_summary table (reaches next $TEXT title with $$)
+ missing_fourth_dd = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " 0 0.1912 0.2315 0.827 12345. 654.3 0.0081 0.410 1.120 0.550 0.045\n $$",
+ " 0 0.1912 0.2315 0.827 12345. 654.3 0.0081 0.410 1.120 0.550 0.045",
+ )
+ with self.assertRaisesRegex(
+ ValueError, r"Multiple \$\$ delimiters on line .* in \$TABLE"
+ ):
+ parse_refmac_log(missing_fourth_dd)
+
+ # 2b. Missing closing data_end $$ before a non-$$ boundary (reaches next $TABLE)
+ missing_resln_fourth_dd = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " 0.150 3432 98.10 21.5 20.4 0.20 0.23 184 20.8 19.5 0.24 0.27\n$$\n\n $TABLE:",
+ " 0.150 3432 98.10 21.5 20.4 0.20 0.23 184 20.8 19.5 0.24 0.27\n $TABLE:",
+ )
+ with self.assertRaisesRegex(ValueError, r"Unclosed data block in \$TABLE"):
+ parse_refmac_log(missing_resln_fourth_dd)
+
+ # 3. Multiple $$ on a single line inside $TABLE
+ double_dd_line = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "$$\nM(4SSQ/LL)",
+ "$$ $$\nM(4SSQ/LL)",
+ )
+ with self.assertRaisesRegex(ValueError, "Multiple \\$\\$ delimiters"):
+ parse_refmac_log(double_dd_line)
+
+ # 4. Trailing tokens after header_end $$
+ trailing_after_header_dd = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Rf_free WR_free $$",
+ "Rf_free WR_free $$ EXTRA_COL",
+ )
+ with self.assertRaisesRegex(ValueError, "Trailing tokens after header_end \\$\\$"):
+ parse_refmac_log(trailing_after_header_dd)
+
+ # 5. Unclosed $TEXT:Result: block
+ unclosed_result = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.rstrip()
+ assert unclosed_result.endswith("$$")
+ unclosed_result = unclosed_result[:-2]
+ with self.assertRaisesRegex(ValueError, "Unclosed \\$TEXT:Result:"):
+ parse_refmac_log(unclosed_result)
+
+ # 6. Stray $$ outside any $TABLE or $TEXT block
+ stray_dd = "$$\n" + SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG
+ with self.assertRaisesRegex(ValueError, "Stray or malformed loggraph delimiter"):
+ parse_refmac_log(stray_dd)
+
+ def test_conflicting_r_pairs_rejected(self) -> None:
+ # 1. Final detailed Overall R factor conflicts with cycle_summary Rfact
+ conflict_detail_work = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ "Overall R factor = 0.5207",
+ "Overall R factor = 0.5208",
+ )
+ with self.assertRaisesRegex(ValueError, "Conflicting R-work"):
+ parse_refmac_log(conflict_detail_work)
+
+ # 2. Final detailed Free R factor conflicts with cycle_summary Rfree
+ conflict_detail_free = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ "Free R factor = 0.5289",
+ "Free R factor = 0.5290",
+ )
+ with self.assertRaisesRegex(ValueError, "Conflicting R-free"):
+ parse_refmac_log(conflict_detail_free)
+
+ # 3. Final results R factor Final conflicts with cycle_summary
+ conflict_final_res_work = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ " R factor 0.5397 0.5207",
+ " R factor 0.5397 0.5210",
+ )
+ with self.assertRaisesRegex(ValueError, "Conflicting R-work"):
+ parse_refmac_log(conflict_final_res_work)
+
+ # 4. Final results R free Final conflicts with cycle_summary
+ conflict_final_res_free = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ " R free 0.5317 0.5289",
+ " R free 0.5317 0.5299",
+ )
+ with self.assertRaisesRegex(ValueError, "Conflicting R-free"):
+ parse_refmac_log(conflict_final_res_free)
+
+ # 5. Final results Initial R factor conflicts with initial cycle (Ncyc 0)
+ conflict_init_res_work = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ " R factor 0.5397 0.5207",
+ " R factor 0.5399 0.5207",
+ )
+ with self.assertRaisesRegex(ValueError, "Conflicting R-work"):
+ parse_refmac_log(conflict_init_res_work)
+
+ # 6. Intermediate cycle (Ncyc 15) mismatch between detail and summary
+ conflict_mid_cycle = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ " 15 0.5302 0.5342",
+ " 15 0.5309 0.5342",
+ )
+ with self.assertRaisesRegex(ValueError, "Conflicting R-work"):
+ parse_refmac_log(conflict_mid_cycle)
+
+ # 7. Unequal Decimal value at 5th decimal place (0.19121 vs 0.1912) rejected with no tolerance
+ conflict_fifth_dp = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Overall R factor = 0.1912",
+ "Overall R factor = 0.19121",
+ )
+ with self.assertRaisesRegex(ValueError, "Conflicting R-work"):
+ parse_refmac_log(conflict_fifth_dp)
+
+ # 8. Missing unweighted Overall R factor (only weighted R present)
+ missing_unweighted = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Overall R factor = 0.1912\n",
+ "",
+ )
+ with self.assertRaisesRegex(
+ ValueError, "missing paired unweighted 'Overall R factor' / 'Free R factor'"
+ ):
+ parse_refmac_log(missing_unweighted)
+
+ def test_decimal_r_reconciliation_with_trailing_zeros_preserves_tokens(self) -> None:
+ # Equal Decimal values with different trailing zeros (0.19120 vs 0.1912 vs 0.191200) pass
+ trailing_zero_log = (
+ SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Overall R factor = 0.1912",
+ "Overall R factor = 0.19120",
+ )
+ .replace(
+ "Free R factor = 0.2315",
+ "Free R factor = 0.231500",
+ )
+ .replace(
+ " R factor 0.1912 0.1912",
+ " R factor 0.191200 0.1912000",
+ )
+ .replace(
+ " R free 0.2315 0.2315",
+ " R free 0.23150 0.2315",
+ )
+ )
+ parsed = parse_refmac_log(trailing_zero_log)
+ self.assertTrue(
+ parsed["cycle_r_values"]["all_ordered_detail_pairs_equal_summary_values"]
+ )
+ self.assertTrue(
+ parsed["consistency_checks"][
+ "initial_detail_summary_and_final_results_initial_values_equal"
+ ]
+ )
+ self.assertTrue(
+ parsed["consistency_checks"][
+ "final_detail_summary_and_final_results_final_values_equal"
+ ]
+ )
+
+ init_cyc = parsed["reconciled_initial_cycle"]
+ self.assertEqual(init_cyc["detailed_r_work_token"], "0.19120")
+ self.assertEqual(init_cyc["detailed_r_work_decimal_places"], 5)
+ self.assertEqual(init_cyc["summary_r_work_token"], "0.1912")
+ self.assertEqual(init_cyc["summary_r_work_decimal_places"], 4)
+ self.assertEqual(init_cyc["final_results_r_work_token"], "0.191200")
+ self.assertEqual(init_cyc["final_results_r_work_decimal_places"], 6)
+
+ final_cyc = parsed["reconciled_final_cycle"]
+ self.assertEqual(final_cyc["detailed_r_free_token"], "0.231500")
+ self.assertEqual(final_cyc["detailed_r_free_decimal_places"], 6)
+ self.assertEqual(final_cyc["summary_r_free_token"], "0.2315")
+ self.assertEqual(final_cyc["summary_r_free_decimal_places"], 4)
+ self.assertEqual(final_cyc["final_results_r_work_token"], "0.1912000")
+ self.assertEqual(final_cyc["final_results_r_work_decimal_places"], 7)
+
+ def test_truncated_rows_and_malformed_headers_rejected(self) -> None:
+ # 1. Truncated row in resolution R-factor table (11 tokens instead of 12)
+ truncated_bin_row = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " 0.150 3432 98.10 21.5 20.4 0.20 0.23 184 20.8 19.5 0.24 0.27",
+ " 0.150 3432 98.10 21.5 20.4 0.20 0.23 184 20.8 19.5 0.24",
+ )
+ with self.assertRaisesRegex(ValueError, "Row width mismatch"):
+ parse_refmac_log(truncated_bin_row)
+
+ # 2. Truncated row in cycle-summary table (10 tokens instead of 11)
+ truncated_summary_row = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " 0 0.1912 0.2315 0.827 12345. 654.3 0.0081 0.410 1.120 0.550 0.045",
+ " 0 0.1912 0.2315 0.827 12345. 654.3 0.0081 0.410 1.120 0.550",
+ )
+ with self.assertRaisesRegex(ValueError, "Row width mismatch"):
+ parse_refmac_log(truncated_summary_row)
+
+ # 3. Truncated row in Final results block (missing Final value for known label 'R free')
+ truncated_final_results = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " R free 0.2315 0.2315",
+ " R free 0.2315",
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ r"Expected exactly 2 values \(Initial, Final\) for 'R free' in Final results block",
+ ):
+ parse_refmac_log(truncated_final_results)
+
+ # 4. Swapped/corrupted header in resolution R-factor table
+ corrupted_header = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Rf_used WR_used",
+ "WR_used Rf_used",
+ )
+ with self.assertRaisesRegex(
+ ValueError, "Unexpected resolution R-factor table header"
+ ):
+ parse_refmac_log(corrupted_header)
+
+ def test_final_results_known_and_other_label_cardinality_and_duplicates(self) -> None:
+ # Extra value on known 'R factor' row rejected
+ extra_rfactor_val = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " R factor 0.1912 0.1912",
+ " R factor 0.1912 0.1912 0.1912",
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ r"Expected exactly 2 values \(Initial, Final\) for 'R factor' in Final results block",
+ ):
+ parse_refmac_log(extra_rfactor_val)
+
+ # Extra value on known 'R free' row rejected
+ extra_rfree_val = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " R free 0.2315 0.2315",
+ " R free 0.2315 0.2315 0.2500",
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ r"Expected exactly 2 values \(Initial, Final\) for 'R free' in Final results block",
+ ):
+ parse_refmac_log(extra_rfree_val)
+
+ # Duplicate 'R factor' row rejected
+ dup_rfactor = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " R factor 0.1912 0.1912",
+ " R factor 0.1912 0.1912\n R factor 0.1912 0.1912",
+ )
+ with self.assertRaisesRegex(
+ ValueError, "Duplicate 'R factor' row in Final results block"
+ ):
+ parse_refmac_log(dup_rfactor)
+
+ # Duplicate 'R free' row rejected
+ dup_rfree = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " R free 0.2315 0.2315",
+ " R free 0.2315 0.2315\n R free 0.2315 0.2315",
+ )
+ with self.assertRaisesRegex(
+ ValueError, "Duplicate 'R free' row in Final results block"
+ ):
+ parse_refmac_log(dup_rfree)
+
+ # Legitimate other two-value label in Final results accepted
+ other_label_ok = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " Rms ChirVolume 0.0450 0.0450",
+ " Rms ChirVolume 0.0450 0.0450\n Rms Planarity 0.0120 0.0080",
+ )
+ parsed_other = parse_refmac_log(other_label_ok)
+ self.assertEqual(parsed_other["final_results_block"]["rows"][-1]["label"], "Rms Planarity")
+
+ # Extra value on other label rejected
+ other_label_extra = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " Rms ChirVolume 0.0450 0.0450",
+ " Rms ChirVolume 0.0450 0.0450\n Rms Planarity 0.0120 0.0080 0.0040",
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ r"Expected exactly 2 values \(Initial, Final\) for 'Rms Planarity' in Final results block",
+ ):
+ parse_refmac_log(other_label_extra)
+
+ def test_nan_infinity_negative_and_malformed_r_values_rejected_while_large_finite_accepted(
+ self,
+ ) -> None:
+ # Large finite ordinary and weighted R values (such as 99.99 or 100.0000) are accepted
+ large_finite_log = (
+ SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace("0.1912", "100.0000")
+ .replace("0.2315", "99.9900")
+ .replace("0.18 0.21", "99.99 100.00")
+ .replace("0.22 0.25", "100.00 120.50")
+ .replace("0.2210", "100.0000")
+ .replace("0.2610", "105.5000")
+ )
+ parsed_large = parse_refmac_log(large_finite_log)
+ self.assertEqual(
+ parsed_large["reconciled_final_cycle"]["r_work_token"], "100.0000"
+ )
+ self.assertEqual(
+ parsed_large["reconciled_final_cycle"]["r_free_token"], "99.9900"
+ )
+ self.assertAlmostEqual(
+ parsed_large["reconciled_final_cycle"]["r_work"], 100.0
+ )
+
+ # Non-finite, negative, overflow ('*'), and malformed R tokens are rejected
+ for bad_token in (
+ "NaN",
+ "nan",
+ "Inf",
+ "-inf",
+ "****",
+ "-0.1912",
+ "N/A",
+ ".1912",
+ "0.",
+ ):
+ with self.subTest(bad_bin_r=bad_token):
+ bad_bin_log = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "0.20 0.23",
+ f"{bad_token} 0.23",
+ )
+ with self.assertRaisesRegex(ValueError, "R token"):
+ parse_refmac_log(bad_bin_log)
+
+ with self.subTest(bad_detail_r=bad_token):
+ bad_detail_log = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Overall R factor = 0.1912",
+ f"Overall R factor = {bad_token}",
+ )
+ with self.assertRaisesRegex(ValueError, "R token"):
+ parse_refmac_log(bad_detail_log)
+
+ with self.subTest(bad_summary_r=bad_token):
+ bad_summary_log = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " 0 0.1912 0.2315",
+ f" 0 {bad_token} 0.2315",
+ )
+ with self.assertRaisesRegex(ValueError, "R token"):
+ parse_refmac_log(bad_summary_log)
+
+ def test_intermediate_resolution_rfactor_table_validated_and_retained(self) -> None:
+ intermediate_table = (
+ "$TABLE: Cycle 2. Rfactor analysis, F distribution v resln :\n"
+ "$GRAPHS:Cycle 2. v. resln :N:1,6,7,11,12:\n"
+ "$$\n"
+ "M(4SSQ/LL) NR_used %_obs M(Fo_used) M(Fc_used) Rf_used WR_used\n"
+ "NR_free M(Fo_free) M(Fc_free) Rf_free WR_free $$\n"
+ "$$\n"
+ " 0.032 11000 99.36 39.1 31.2 0.53 0.76 600 24.6 36.2 0.75 1.45\n"
+ " 0.093 11467 100.00 35.1 25.7 0.52 0.60 613 32.9 26.5 0.52 0.58\n"
+ "$$\n"
+ "Number of used reflections = 22467\n"
+ )
+ anchor = (
+ "Overall R factor = 0.5380\n"
+ "Free R factor = 0.5313\n"
+ )
+ with_intermediate = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ anchor, intermediate_table + anchor, 1
+ )
+ parsed = parse_refmac_log(with_intermediate)
+ tables = parsed["resolution_bin_values"]["tables"]
+ self.assertEqual(len(tables), 3)
+ self.assertEqual(
+ [t["phase"] for t in tables], ["initial", "intermediate", "final"]
+ )
+ self.assertEqual([t["detailed_cycle"] for t in tables], [1, 2, 31])
+ self.assertEqual(tables[1]["summary_ncyc"], 1)
+ self.assertEqual(tables[1]["nr_used_sum"], 22467)
+ self.assertEqual(tables[1]["nr_free_sum"], 1213)
+ self.assertEqual(parsed["initial_resolution_bins"]["detailed_cycle"], 1)
+ self.assertEqual(parsed["final_resolution_bins"]["detailed_cycle"], 31)
+
+ # 1. NaN R token inside intermediate resolution table rejected
+ bad_intermediate_r = with_intermediate.replace(
+ "0.53 0.76 600", "NaN 0.76 600", 1
+ )
+ with self.assertRaisesRegex(ValueError, "R token"):
+ parse_refmac_log(bad_intermediate_r)
+
+ # 2. Negative and fractional count inside intermediate resolution table rejected
+ neg_intermediate_count = with_intermediate.replace(
+ " 0.032 11000", " 0.032 -11000", 1
+ )
+ with self.assertRaisesRegex(ValueError, "NR_used"):
+ parse_refmac_log(neg_intermediate_count)
+
+ frac_intermediate_count = with_intermediate.replace(
+ " 0.032 11000", " 0.032 11000.5", 1
+ )
+ with self.assertRaisesRegex(ValueError, "NR_used"):
+ parse_refmac_log(frac_intermediate_count)
+
+ # 3. Same-cycle detailed Number of used reflections mismatch against intermediate table sum rejected
+ mismatch_intermediate_used = with_intermediate.replace(
+ "Number of used reflections = 22467\nOverall R factor = 0.5380",
+ "Number of used reflections = 22468\nOverall R factor = 0.5380",
+ 1,
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "Detailed Cycle 2 Number of used reflections .* != Cycle 2 resolution table NR_used sum",
+ ):
+ parse_refmac_log(mismatch_intermediate_used)
+
+ def test_cgmat_cycle_markers_validated_and_retained(self) -> None:
+ # Optional absence in HISTORICAL_AMPLE_EXCERPT_LOG
+ parsed_no_markers = parse_refmac_log(HISTORICAL_AMPLE_EXCERPT_LOG)
+ self.assertEqual(parsed_no_markers["cgmat_cycle_markers"], [])
+ self.assertIsNone(parsed_no_markers["detailed_cycles"][0]["cgmat_cycle_number"])
+
+ # Native pattern: CGMAT cycle number 1..30 before detailed cycles 1..30, no marker 31 before detailed 31
+ lines = HISTORICAL_AMPLE_EXCERPT_LOG.splitlines()
+ out_lines: list[str] = []
+ overall_r_seen = 0
+ for line in lines:
+ if line.startswith("Resolution limits") and overall_r_seen == 0:
+ out_lines.append(" CGMAT cycle number = 1")
+ elif line.startswith("Overall R factor"):
+ overall_r_seen += 1
+ if 2 <= overall_r_seen <= 30:
+ out_lines.append(
+ f" CGMAT cycle number = {overall_r_seen:2d}"
+ )
+ out_lines.append(line)
+ log_with_30_markers = "\n".join(out_lines) + "\n"
+ parsed_markers = parse_refmac_log(log_with_30_markers)
+ self.assertEqual(len(parsed_markers["cgmat_cycle_markers"]), 30)
+ self.assertEqual(
+ [m["cycle_number"] for m in parsed_markers["cgmat_cycle_markers"]],
+ list(range(1, 31)),
+ )
+ self.assertEqual(
+ [m["detailed_cycle"] for m in parsed_markers["cgmat_cycle_markers"]],
+ list(range(1, 31)),
+ )
+ self.assertEqual(parsed_markers["detailed_cycles"][0]["cgmat_cycle_number"], 1)
+ self.assertEqual(
+ parsed_markers["detailed_cycles"][29]["cgmat_cycle_number"], 30
+ )
+ self.assertIsNone(parsed_markers["detailed_cycles"][30]["cgmat_cycle_number"])
+
+ # Duplicate CGMAT marker rejected
+ dup_marker_log = log_with_30_markers.replace(
+ " CGMAT cycle number = 2",
+ " CGMAT cycle number = 1",
+ 1,
+ )
+ with self.assertRaisesRegex(ValueError, "CGMAT cycle number"):
+ parse_refmac_log(dup_marker_log)
+
+ # Out-of-order / misassociated CGMAT marker rejected
+ misassociated_marker = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Resolution limits",
+ " CGMAT cycle number = 2\nResolution limits",
+ )
+ with self.assertRaisesRegex(
+ ValueError, "CGMAT cycle number 2 .* does not match associated detailed Cycle 1"
+ ):
+ parse_refmac_log(misassociated_marker)
+
+ # Non-positive / non-finite CGMAT marker rejected
+ for bad_cgmat in ("0", "-1", "1.5", "NaN", "1 2"):
+ with self.subTest(bad_cgmat=bad_cgmat):
+ bad_cgmat_log = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Resolution limits",
+ f" CGMAT cycle number = {bad_cgmat}\nResolution limits",
+ )
+ with self.assertRaisesRegex(ValueError, "CGMAT cycle number"):
+ parse_refmac_log(bad_cgmat_log)
+
+ def test_optional_correlation_dpi_and_ml_su_fields_validated(self) -> None:
+ parsed = parse_refmac_log(HISTORICAL_AMPLE_EXCERPT_LOG)
+ init_detail = parsed["detailed_cycles"][0]
+ final_detail = parsed["detailed_cycles"][-1]
+
+ # Absent on Cycle 1 in excerpt
+ self.assertIsNone(init_detail["overall_correlation_coefficient"])
+ self.assertIsNone(init_detail["free_correlation_coefficient"])
+ self.assertIsNone(init_detail["cruickshanks_dpi"])
+ self.assertIsNone(init_detail["dpi_free_r"])
+ self.assertIsNone(init_detail["ml_su_positional"])
+ self.assertIsNone(init_detail["ml_su_thermal"])
+
+ # Present on Cycle 31 with tokens, values, and source line numbers
+ self.assertEqual(
+ final_detail["overall_correlation_coefficient_token"], "0.6076"
+ )
+ self.assertAlmostEqual(
+ final_detail["overall_correlation_coefficient"], 0.6076
+ )
+ self.assertIsNotNone(final_detail["overall_correlation_coefficient_line"])
+ self.assertEqual(final_detail["free_correlation_coefficient_token"], "0.5684")
+ self.assertEqual(final_detail["cruickshanks_dpi_token"], "0.0241")
+ self.assertEqual(final_detail["dpi_free_r_token"], "0.0244")
+ self.assertEqual(final_detail["ml_su_positional_token"], "0.0589")
+ self.assertEqual(final_detail["ml_su_thermal_token"], "1.0064")
+
+ # Non-finite or wrong-cardinality optional metric values rejected
+ for label, orig_val in (
+ ("Overall correlation coefficient", "0.6076"),
+ ("Free correlation coefficient", "0.5684"),
+ ("Cruickshanks DPI for coordinate error", "0.0241"),
+ ("DPI based on free R factor", "0.0244"),
+ ("ML based su of positional parameters", "0.0589"),
+ ("ML based su of thermal parameters", "1.0064"),
+ ):
+ for bad_rhs in ("NaN", "Inf", "****", f"{orig_val} {orig_val}", ""):
+ with self.subTest(label=label, bad_rhs=bad_rhs):
+ mutated = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ orig_val, bad_rhs, 1
+ )
+ with self.assertRaises(ValueError):
+ parse_refmac_log(mutated)
+
+ def test_fom_resolution_tables_cycle_order_and_row_count_identity(self) -> None:
+ # FOM population may differ from R-table NR_used / NR_free as long as NREFall == NREFa + NREFc
+ different_fom_pop = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " 0.0500 3000 0.850 100 0.780 3100 0.848 0.910",
+ " 0.0500 400 0.850 50 0.780 450 0.848 0.910",
+ )
+ parsed = parse_refmac_log(different_fom_pop)
+ self.assertEqual(parsed["reconciled_final_cycle"]["nr_used_sum"], 6532)
+
+ # Within-row NREFall != NREFa + NREFc rejected
+ bad_fom_sum = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " 0.0500 3000 0.850 100 0.780 3100 0.848 0.910",
+ " 0.0500 3000 0.850 100 0.780 3101 0.848 0.910",
+ )
+ with self.assertRaisesRegex(
+ ValueError, r"FOM table row NREFall \(3101\) != NREFa \(3000\) \+ NREFc \(100\)"
+ ):
+ parse_refmac_log(bad_fom_sum)
+
+ # Duplicate / non-increasing FOM cycle number rejected
+ dup_fom_cycle = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ " $TABLE: Cycle 31. Fom(-acentric, centric, overall v resln:\n"
+ " $GRAPHS:Cycle 31. M(Fom) v. resln :N:1,3,5,7,8:",
+ " $TABLE: Cycle 1. Fom(-acentric, centric, overall v resln:\n"
+ " $GRAPHS:Cycle 1. M(Fom) v. resln :N:1,3,5,7,8:",
+ )
+ with self.assertRaisesRegex(
+ ValueError, "Non-increasing or duplicate resolution FOM table Cycle 1"
+ ):
+ parse_refmac_log(dup_fom_cycle)
+
+ # Misaligned FOM table cycle number rejected
+ misaligned_fom_cycle = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ " $TABLE: Cycle 31. Fom(-acentric, centric, overall v resln:\n"
+ " $GRAPHS:Cycle 31. M(Fom) v. resln :N:1,3,5,7,8:",
+ " $TABLE: Cycle 30. Fom(-acentric, centric, overall v resln:\n"
+ " $GRAPHS:Cycle 30. M(Fom) v. resln :N:1,3,5,7,8:",
+ )
+ with self.assertRaisesRegex(
+ ValueError, "Resolution FOM table Cycle 30 .* is misaligned"
+ ):
+ parse_refmac_log(misaligned_fom_cycle)
+
+ def test_percentage_tokens_domain_and_precision(self) -> None:
+ # 0 and 100 boundaries accepted; 0.0000 free percentage with positive NR_free accepted
+ boundary_pct_log = (
+ SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Percentage observed = 98.7500",
+ "Percentage observed = 100.0000",
+ )
+ .replace(
+ "Percentage of free reflections = 5.0029",
+ "Percentage of free reflections = 0.0000",
+ )
+ .replace("99.50", "0.00")
+ .replace("98.10", "100.00")
+ )
+ parsed = parse_refmac_log(boundary_pct_log)
+ detail = parsed["detailed_cycles"][0]
+ self.assertEqual(detail["percentage_observed_token"], "100.0000")
+ self.assertAlmostEqual(detail["percentage_observed"], 100.0)
+ self.assertEqual(detail["percentage_observed_decimal_places"], 4)
+ self.assertEqual(detail["percentage_free_token"], "0.0000")
+ self.assertAlmostEqual(detail["percentage_free"], 0.0)
+ self.assertEqual(detail["percentage_free_decimal_places"], 4)
+ self.assertGreater(parsed["final_resolution_bins"]["nr_free_sum"], 0)
+
+ # Scientific-notation percentages compute effective decimal places after exponent
+ exp_pct_log = (
+ SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Percentage observed = 98.7500",
+ "Percentage observed = 8.58624e1",
+ )
+ .replace(
+ "Percentage of free reflections = 5.0029",
+ "Percentage of free reflections = 5.12250e0",
+ )
+ .replace("99.50", "1e2", 1)
+ )
+ parsed_exp = parse_refmac_log(exp_pct_log)
+ exp_detail = parsed_exp["detailed_cycles"][0]
+ self.assertEqual(exp_detail["percentage_observed_token"], "8.58624e1")
+ self.assertAlmostEqual(exp_detail["percentage_observed"], 85.8624)
+ self.assertEqual(exp_detail["percentage_observed_decimal_places"], 4)
+ self.assertEqual(exp_detail["percentage_free_token"], "5.12250e0")
+ self.assertAlmostEqual(exp_detail["percentage_free"], 5.1225)
+ self.assertEqual(exp_detail["percentage_free_decimal_places"], 5)
+ self.assertEqual(
+ parsed_exp["final_resolution_bins"]["bins"][0]["pct_obs_token"], "1e2"
+ )
+ self.assertAlmostEqual(
+ parsed_exp["final_resolution_bins"]["bins"][0]["pct_obs"], 100.0
+ )
+ self.assertEqual(
+ parsed_exp["final_resolution_bins"]["bins"][0]["pct_obs_decimal_places"],
+ 0,
+ )
+
+ # Out-of-domain (< 0, > 100) and non-finite percentages rejected
+ for bad_pct in ("-0.01", "100.01", "NaN", "Inf", "****"):
+ with self.subTest(bad_pct_obs=bad_pct):
+ bad_obs = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Percentage observed = 98.7500",
+ f"Percentage observed = {bad_pct}",
+ )
+ with self.assertRaisesRegex(ValueError, "Percentage observed"):
+ parse_refmac_log(bad_obs)
+
+ with self.subTest(bad_pct_free=bad_pct):
+ bad_free = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Percentage of free reflections = 5.0029",
+ f"Percentage of free reflections = {bad_pct}",
+ )
+ with self.assertRaisesRegex(
+ ValueError, "Percentage of free reflections"
+ ):
+ parse_refmac_log(bad_free)
+
+ with self.subTest(bad_bin_pct=bad_pct):
+ bad_bin = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "99.50", bad_pct, 1
+ )
+ with self.assertRaisesRegex(ValueError, "%_obs"):
+ parse_refmac_log(bad_bin)
+
+ def test_underscore_and_non_native_numeric_literals_rejected(self) -> None:
+ # Underscores in percentage, generic float, integer count, and R fields must fail
+ for target, replacement, err_re in (
+ (
+ "Percentage observed = 85.8624",
+ "Percentage observed = 8_5.8624",
+ "Malformed numeric token '8_5.8624'",
+ ),
+ (
+ " 0.032 293 99.36",
+ " 0.032 293 9_9.36",
+ "Malformed numeric token '9_9.36'",
+ ),
+ (
+ "Overall correlation coefficient = 0.6076",
+ "Overall correlation coefficient = 0_0.6076",
+ "Malformed numeric token '0_0.6076'",
+ ),
+ (
+ "Overall weighted R factor = 0.5425",
+ "Overall weighted R factor = 0_0.5425",
+ "Malformed numeric token '0_0.5425'",
+ ),
+ (
+ " 30 0.5207 0.5289 0.341 65565.",
+ " 30 0.5207 0.5289 0.341 65_565.",
+ "Malformed numeric token '65_565.'",
+ ),
+ (
+ "Number of used reflections = 22467",
+ "Number of used reflections = 22_467",
+ "Invalid non-negative integer count token '22_467'",
+ ),
+ (
+ "Overall R factor = 0.5207",
+ "Overall R factor = 0_0.5207",
+ "Invalid or nonfinite/sentinel R token '0_0.5207'",
+ ),
+ ):
+ with self.subTest(replacement=replacement):
+ mutated = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ target, replacement, 1
+ )
+ with self.assertRaisesRegex(ValueError, err_re):
+ parse_refmac_log(mutated)
+
+ def test_graph_declaration_cycle_labels_validated_against_table_title(self) -> None:
+ # 1. Final resolution R-factor $GRAPHS Cycle 31 -> Cycle 30 contradicts table title Cycle 31
+ contradict_r_graph = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ "$GRAPHS:Cycle 31. v. resln :N:1,6,7,11,12:",
+ "$GRAPHS:Cycle 30. v. resln :N:1,6,7,11,12:",
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ r"Conflicting \$GRAPHS Cycle 30 .* vs resolution R-factor table title Cycle 31",
+ ):
+ parse_refmac_log(contradict_r_graph)
+
+ # 2. Final resolution FOM $GRAPHS Cycle 31 -> Cycle 30 contradicts table title Cycle 31
+ contradict_fom_graph = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ " $GRAPHS:Cycle 31. M(Fom) v. resln :N:1,3,5,7,8:",
+ " $GRAPHS:Cycle 30. M(Fom) v. resln :N:1,3,5,7,8:",
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ r"Conflicting \$GRAPHS Cycle 30 .* vs resolution FOM table title Cycle 31",
+ ):
+ parse_refmac_log(contradict_fom_graph)
+
+ # 3. Continuation line :Cycle 31. -> :Cycle 30. contradicts table title Cycle 31
+ contradict_cont_graph = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ ":Cycle 31. and v. resln :N:1,4,5,9,10:",
+ ":Cycle 30. and v. resln :N:1,4,5,9,10:",
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ r"Conflicting \$GRAPHS Cycle 30 .* vs resolution R-factor table title Cycle 31",
+ ):
+ parse_refmac_log(contradict_cont_graph)
+
+ # 4. Malformed explicit Cycle labels in $GRAPHS or continuation lines rejected
+ for bad_graph_line in (
+ ":Cycle 31 and v. resln :N:1,4,5,9,10:",
+ ":cycle 31. and v. resln :N:1,4,5,9,10:",
+ ":Cycle NaN. and v. resln :N:1,4,5,9,10:",
+ ":Cycle 0. and v. resln :N:1,4,5,9,10:",
+ ):
+ with self.subTest(bad_graph_line=bad_graph_line):
+ mutated = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ ":Cycle 31. and v. resln :N:1,4,5,9,10:",
+ bad_graph_line,
+ )
+ with self.assertRaisesRegex(ValueError, r"\$GRAPHS"):
+ parse_refmac_log(mutated)
+
+ # 5. Valid graph declaration without an explicit Cycle label is accepted
+ no_cycle_in_graph = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "$GRAPHS:Cycle 1. v. resln :N:1,6,7,11,12:",
+ "$GRAPHS: v. resln :N:1,6,7,11,12:",
+ )
+ parsed_no_graph_cycle = parse_refmac_log(no_cycle_in_graph)
+ self.assertEqual(
+ parsed_no_graph_cycle["reconciled_final_cycle"]["resolution_table_cycle"],
+ 1,
+ )
+
+ def test_fractional_negative_and_contradictory_counts_rejected(self) -> None:
+ # 1. Fractional NR_used
+ frac_nr_used = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " 0.050 3100",
+ " 0.050 3100.5",
+ )
+ with self.assertRaisesRegex(ValueError, "NR_used"):
+ parse_refmac_log(frac_nr_used)
+
+ # 2. Fractional NR_free
+ frac_nr_free = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " 160 41.2",
+ " 160.0 41.2",
+ )
+ with self.assertRaisesRegex(ValueError, "NR_free"):
+ parse_refmac_log(frac_nr_free)
+
+ # 3. Negative NR_used
+ neg_nr_used = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " 0.050 3100",
+ " 0.050 -3100",
+ )
+ with self.assertRaisesRegex(ValueError, "NR_used"):
+ parse_refmac_log(neg_nr_used)
+
+ # 4. Negative NR_free
+ neg_nr_free = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " 160 41.2",
+ " -1 41.2",
+ )
+ with self.assertRaisesRegex(ValueError, "NR_free"):
+ parse_refmac_log(neg_nr_free)
+
+ # 5. Fractional summary Ncyc
+ frac_ncyc = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " 0 0.1912 0.2315",
+ " 0.0 0.1912 0.2315",
+ )
+ with self.assertRaisesRegex(ValueError, "summary Ncyc"):
+ parse_refmac_log(frac_ncyc)
+
+ # 6. Contradictory Number of used reflections vs sum(NR_used)
+ contradictory_used = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ "Number of used reflections = 6532",
+ "Number of used reflections = 6533",
+ )
+ with self.assertRaisesRegex(
+ ValueError, "Number of used reflections .* != .* NR_used sum"
+ ):
+ parse_refmac_log(contradictory_used)
+
+ def test_ambiguous_final_cycle_rejected(self) -> None:
+ # 1. Wrong final resolution table cycle label (Cycle 30 instead of Cycle 31)
+ wrong_final_table_cycle = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ "$TABLE: Cycle 31. Rfactor analysis, F distribution v resln :\n"
+ "$GRAPHS:Cycle 31. v. resln :N:1,6,7,11,12:\n"
+ ":Cycle 31. and v. resln :N:1,4,5,9,10:\n"
+ ":Cycle 31. % observed v. resln :N:1,3:",
+ "$TABLE: Cycle 30. Rfactor analysis, F distribution v resln :\n"
+ "$GRAPHS:Cycle 30. v. resln :N:1,6,7,11,12:\n"
+ ":Cycle 30. and v. resln :N:1,4,5,9,10:\n"
+ ":Cycle 30. % observed v. resln :N:1,3:",
+ )
+ with self.assertRaisesRegex(ValueError, "Resolution R-factor table Cycle 30"):
+ parse_refmac_log(wrong_final_table_cycle)
+
+ # 2. Missing final resolution R-factor table in multi-cycle log
+ parts = HISTORICAL_AMPLE_EXCERPT_LOG.split(
+ "$TABLE: Cycle 31. Rfactor analysis, F distribution v resln :"
+ )
+ self.assertEqual(len(parts), 2)
+ only_initial_resln_table = (
+ parts[0]
+ + "$TABLE: Cycle 31. Unrelated stats v resln :"
+ + parts[1].replace(
+ "M(4SSQ/LL) NR_used %_obs M(Fo_used) M(Fc_used) Rf_used WR_used\n"
+ "NR_free M(Fo_free) M(Fc_free) Rf_free WR_free $$",
+ "C1 C2 C3 C4 C5 C6 C7 C8 C9 C10 C11 C12 $$",
+ 1,
+ )
+ )
+ with self.assertRaisesRegex(
+ ValueError, "Ambiguous or missing final resolution R-factor table"
+ ):
+ parse_refmac_log(only_initial_resln_table)
+
+ # 3. Extra detailed Overall/Free R pair not in cycle_summary
+ extra_detail_pair = HISTORICAL_AMPLE_EXCERPT_LOG.replace(
+ "$TABLE: Rfactor analysis, stats vs cycle :",
+ "Overall R factor = 0.5207\nFree R factor = 0.5289\n"
+ "-----------------------------------------------------------------------------\n"
+ "$TABLE: Rfactor analysis, stats vs cycle :",
+ )
+ with self.assertRaisesRegex(ValueError, "Cycle count mismatch"):
+ parse_refmac_log(extra_detail_pair)
+
+ # 4. Contradictory declared NCYC vs summary max Ncyc
+ contradictory_ncyc = SYNTHETIC_SINGLE_CYCLE_NCYC0_LOG.replace(
+ " Data line--- NCYC 0\n Refinement cycles : 0",
+ " Data line--- NCYC 5\n Refinement cycles : 5",
+ )
+ with self.assertRaisesRegex(
+ ValueError, "Cycle summary max Ncyc .* contradicts declared NCYC"
+ ):
+ parse_refmac_log(contradictory_ncyc)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/test_onesar_rerun.py b/scripts/test_onesar_rerun.py
new file mode 100644
index 0000000..7218692
--- /dev/null
+++ b/scripts/test_onesar_rerun.py
@@ -0,0 +1,5806 @@
+#!/usr/bin/env python3
+"""Hermetic unit tests for 1SAR input admission, MTZ projection, and PHENIX R-factor reconciliation.
+
+Runs under the repository Python environment with zero PHENIX/CCP4/Gemmi
+execution, zero vendor library imports, and zero network access.
+"""
+
+from __future__ import annotations
+
+import ast
+import copy
+import hashlib
+import io
+import json
+import math
+import os
+import struct
+import subprocess
+import sys
+import tempfile
+import time
+import types
+import unittest
+import warnings
+import zipfile
+from pathlib import Path
+from typing import Any
+from unittest import mock
+
+_SCRIPTS_DIR = Path(__file__).resolve().parent
+if str(_SCRIPTS_DIR) not in sys.path:
+ sys.path.insert(0, str(_SCRIPTS_DIR))
+
+import onesar_phenix_measure # noqa: E402
+import onesar_rerun # noqa: E402
+import refinement_cache # noqa: E402
+import toolchain # noqa: E402
+
+
+REPO_ROOT = _SCRIPTS_DIR.parent
+COMMITTED_ARCHIVE = REPO_ROOT / onesar_rerun.ARCHIVE_REL_PATH
+
+
+def _card(text: str) -> bytes:
+ return text.ljust(80).encode("ascii")
+
+
+def _build_synthetic_mtz(
+ rows: list[tuple[float, ...]],
+ *,
+ columns: list[tuple[str, str, str]] | None = None,
+ datasets: list[tuple[int, str]] | None = None,
+ cell: tuple[float, float, float, float, float, float] = (
+ 64.897,
+ 78.323,
+ 38.792,
+ 90.0,
+ 90.0,
+ 90.0,
+ ),
+ syminf: str = "SYMINF 4 4 P 19 'P 21 21 21' PG222",
+ symm_cards: tuple[str, ...] = (
+ "SYMM X,Y,Z",
+ "SYMM -X+1/2,-Y,Z+1/2",
+ "SYMM -X,Y+1/2,-Z+1/2",
+ "SYMM X+1/2,-Y+1/2,-Z",
+ ),
+ valm: str = "VALM NAN",
+) -> bytes:
+ """Construct a synthetic little-endian MTZ byte stream for pure unit testing."""
+ if columns is None:
+ columns = [
+ ("H", "H", "1"),
+ ("K", "H", "1"),
+ ("L", "H", "1"),
+ ("F-obs", "F", "1"),
+ ("SIGF-obs", "Q", "1"),
+ ("R-free-flags", "I", "1"),
+ ]
+ if datasets is None:
+ datasets = [(1, "Original-experimental-data-mapped-to-asu")]
+ ncols = len(columns)
+ nrows = len(rows)
+ flat_values: list[float] = []
+ for row in rows:
+ if len(row) != ncols:
+ raise ValueError(f"Row width {len(row)} != ncols {ncols}")
+ flat_values.extend(float(v) for v in row)
+ data_bytes = struct.pack(f"<{len(flat_values)}f", *flat_values)
+ header_offset = (80 + len(data_bytes)) // 4 + 1
+
+ cards: list[bytes] = [
+ _card("VERS MTZ:V1.1"),
+ _card("TITLE synthetic test mtz"),
+ _card(f"NCOL {ncols:8d} {nrows:12d} {0:8d}"),
+ _card(
+ f"CELL {cell[0]:9.4f} {cell[1]:9.4f} {cell[2]:9.4f} "
+ f"{cell[3]:8.4f} {cell[4]:8.4f} {cell[5]:8.4f}"
+ ),
+ _card("SORT 1 2 3 0 0"),
+ _card(syminf),
+ ]
+ for symm in symm_cards:
+ cards.append(_card(symm))
+ cards.append(_card("RESO 0.00200000 0.25000000"))
+ cards.append(_card(valm))
+ cards.append(_card(f"NDIF {len(datasets):8d}"))
+ for d_id, d_name in datasets:
+ cards.append(_card(f"PROJECT {d_id:7d} project"))
+ cards.append(_card(f"CRYSTAL {d_id:7d} crystal"))
+ cards.append(_card(f"DATASET {d_id:7d} {d_name}"))
+ cards.append(
+ _card(
+ f"DCELL {d_id:9d} {cell[0]:9.4f} {cell[1]:9.4f} {cell[2]:9.4f} "
+ f"{cell[3]:8.4f} {cell[4]:8.4f} {cell[5]:8.4f}"
+ )
+ )
+ cards.append(_card(f"DWAVEL {d_id:8d} 1.00000"))
+ for name, ctype, d_id in columns:
+ cards.append(_card(f"COLUMN {name:<30s} {ctype} 0.0 100.0 {d_id}"))
+ cards.append(_card("END"))
+
+ preamble = (
+ b"MTZ "
+ + struct.pack(" tuple[bytes, bytes]:
+ """Build synthetic phx_maps.mtz and PHX_OBS_PAIR_ONLY.mtz byte streams."""
+ fofc_by_hkl = {
+ tuple(rec["hkl"]): rec for rec in records_fofc
+ }
+ maps_rows: list[tuple[float, ...]] = []
+ pair_rows: list[tuple[float, ...]] = []
+ for rec2 in records_2fofc:
+ h, k, l = rec2["hkl"]
+ rec1 = fofc_by_hkl[(h, k, l)]
+ # Emulate iotbx.mtz phase storage in [0, 360) degrees float32
+ ph2 = float(rec2["phase_deg"]) % 360.0
+ ph1 = float(rec1["phase_deg"]) % 360.0
+ maps_rows.append(
+ (
+ float(h),
+ float(k),
+ float(l),
+ float(rec2["amplitude"]),
+ ph2,
+ float(rec1["amplitude"]),
+ ph1,
+ )
+ )
+ pair_rows.append(
+ (
+ float(h),
+ float(k),
+ float(l),
+ float(rec2["amplitude"]),
+ ph2,
+ )
+ )
+ maps_bytes = _build_synthetic_mtz(
+ maps_rows,
+ columns=[
+ ("H", "H", "1"),
+ ("K", "H", "1"),
+ ("L", "H", "1"),
+ ("2FOFCWT", "F", "1"),
+ ("PH2FOFCWT", "P", "1"),
+ ("FOFCWT", "F", "1"),
+ ("PHFOFCWT", "P", "1"),
+ ],
+ cell=cell,
+ )
+ pair_bytes = _build_synthetic_mtz(
+ pair_rows,
+ columns=[
+ ("H", "H", "1"),
+ ("K", "H", "1"),
+ ("L", "H", "1"),
+ ("2FOFCWT", "F", "1"),
+ ("PH2FOFCWT", "P", "1"),
+ ],
+ cell=cell,
+ )
+ return maps_bytes, pair_bytes
+
+
+def _make_reconciliation_fixture() -> tuple[
+ dict[str, Any], dict[str, Any], dict[str, str], bytes, bytes
+]:
+ """Create a pure-math synthetic input ledger, helper payload, and map MTZs."""
+ nan = float("nan")
+ mtz_bytes = _build_synthetic_mtz(
+ [
+ # Eligible work rows (flag=0), including one zero-F and one negative-F row
+ (1, 0, 0, 100.0, 2.0, 0.0),
+ (0, 1, 0, 200.0, 3.0, 0.0),
+ (0, 0, 1, 0.0, 1.5, 0.0),
+ (1, 1, 0, -5.0, 1.2, 0.0),
+ (1, 1, 1, 150.0, 2.5, 0.0),
+ # Eligible free rows (flag=1)
+ (2, 0, 0, 80.0, 1.8, 1.0),
+ (0, 2, 0, 120.0, 2.2, 1.0),
+ # Ineligible all-NaN padding row (F, SIGF, flag all NaN)
+ (0, 0, 2, nan, nan, nan),
+ ]
+ )
+ input_ledger = onesar_rerun.build_input_hkl_ledger(mtz_bytes)
+
+ # Suppose extract_xtal_data regularizes out the zero-F (0,0,1) and negative-F (1,1,0)
+ # rows, leaving 5 extracted reflections; and f_model outlier removal drops (1,1,1),
+ # leaving 2 work reflections [(1,0,0), (0,1,0)] and 2 free reflections [(2,0,0), (0,2,0)].
+ scale_k1_w = 0.5
+ scale_k1_t = 0.6
+ scale_k1 = 0.54
+
+ # Work: (1,0,0) fo=100, |fm_raw|=180 -> |fm_scaled_w|=90; diff=10
+ # (0,1,0) fo=200, |fm_raw|=420 -> |fm_scaled_w|=210; diff=10
+ # work_num = 20, work_den = 300, r_work = 20 / 300
+ # Free: (2,0,0) fo=80, |fm_raw|=100 -> |fm_scaled_t|=60; diff=20
+ # (0,2,0) fo=120, |fm_raw|=250 -> |fm_scaled_t|=150; diff=30
+ # free_num = 50, free_den = 200, r_free = 50 / 200 = 0.25
+ # All (using scale_k1 = 0.54):
+ # (1,0,0): |100 - 0.54*180| = |100 - 97.2| = 2.8
+ # (0,1,0): |200 - 0.54*420| = |200 - 226.8| = 26.8
+ # (2,0,0): |80 - 0.54*100| = |80 - 54.0| = 26.0
+ # (0,2,0): |120 - 0.54*250| = |120 - 135.0| = 15.0
+ # all_num = 70.6, all_den = 500.0, r_all = 70.6 / 500.0 = 0.1412
+ r_work = 20.0 / 300.0
+ r_free = 50.0 / 200.0
+ all_num = 70.6
+ all_den = 500.0
+ r_all = all_num / all_den
+
+ vendor_shas = {
+ key: hashlib.sha256(key.encode("utf-8")).hexdigest()
+ for key in onesar_rerun.PHENIX_VENDOR_SOURCE_RELPATHS
+ }
+ vendor_modules = {
+ key: {
+ "module_name": key,
+ "configured_path": f"/opt/phenix/lib/python3.9/site-packages/{rel}",
+ "resolved_path": f"/opt/phenix/lib/python3.9/site-packages/{rel}",
+ "sha256": vendor_shas[key],
+ }
+ for key, rel in onesar_rerun.PHENIX_VENDOR_SOURCE_RELPATHS.items()
+ }
+
+ scatterer_records = [
+ {
+ "label": "PDB=\" N VAL A 1 \"",
+ "scattering_type": "N",
+ "site_frac": [0.1, 0.2, 0.3],
+ "site_cart": [6.4897, 15.6646, 11.6376],
+ "occupancy": 1.0,
+ "u_iso": 0.25,
+ "u_star": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
+ "use_u_iso": True,
+ "use_u_aniso": False,
+ },
+ {
+ "label": "PDB=\" CA VAL A 1 \"",
+ "scattering_type": "C",
+ "site_frac": [0.12, 0.22, 0.32],
+ "site_cart": [7.78764, 17.23106, 12.41344],
+ "occupancy": 1.0,
+ "u_iso": 0.28,
+ "u_star": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
+ "use_u_iso": True,
+ "use_u_aniso": False,
+ },
+ ]
+ snap = onesar_phenix_measure.snapshot_scatterer_records(scatterer_records)
+ xrs_invariance = onesar_phenix_measure.compare_scatterer_snapshots(
+ snap, copy.deepcopy(snap), copy.deepcopy(snap)
+ )
+
+ final_hkls = [
+ ([1, 0, 0], 100.0, 2.0, False),
+ ([0, 1, 0], 200.0, 3.0, False),
+ ([2, 0, 0], 80.0, 1.8, True),
+ ([0, 2, 0], 120.0, 2.2, True),
+ ]
+ map_records_2fofc = [
+ {
+ "hkl": [1, 0, 0],
+ "real": 10.0,
+ "imag": 0.0,
+ "amplitude": 10.0,
+ "phase_deg": 0.0,
+ "is_centric": True,
+ },
+ {
+ "hkl": [0, 1, 0],
+ "real": -12.0,
+ "imag": 0.0,
+ "amplitude": 12.0,
+ "phase_deg": 180.0,
+ "is_centric": True,
+ },
+ {
+ "hkl": [2, 0, 0],
+ "real": 0.0,
+ "imag": 8.0,
+ "amplitude": 8.0,
+ "phase_deg": 90.0,
+ "is_centric": False,
+ },
+ {
+ "hkl": [0, 2, 0],
+ "real": 0.0,
+ "imag": -6.0,
+ "amplitude": 6.0,
+ "phase_deg": -90.0,
+ "is_centric": False,
+ },
+ ]
+ map_records_fofc = [
+ {
+ "hkl": [1, 0, 0],
+ "real": 2.0,
+ "imag": 0.0,
+ "amplitude": 2.0,
+ "phase_deg": 0.0,
+ "is_centric": True,
+ },
+ {
+ "hkl": [0, 1, 0],
+ "real": -1.5,
+ "imag": 0.0,
+ "amplitude": 1.5,
+ "phase_deg": 180.0,
+ "is_centric": True,
+ },
+ {
+ "hkl": [2, 0, 0],
+ "real": 0.0,
+ "imag": 1.25,
+ "amplitude": 1.25,
+ "phase_deg": 90.0,
+ "is_centric": False,
+ },
+ {
+ "hkl": [0, 2, 0],
+ "real": 0.0,
+ "imag": -0.75,
+ "amplitude": 0.75,
+ "phase_deg": -90.0,
+ "is_centric": False,
+ },
+ ]
+ maps_mtz_bytes, obs_pair_mtz_bytes = _build_synthetic_map_mtz_pair(
+ map_records_2fofc, map_records_fofc
+ )
+ map_params = {
+ "acentrics_scale": 2.0,
+ "centrics_pre_scale": 1.0,
+ "exclude_free_r_reflections": False,
+ "fill_missing": False,
+ "isotropize": True,
+ "sharp": False,
+ }
+
+ runtime_cell = onesar_rerun.ccp4_mtz_float32_unit_cell(
+ input_ledger["crystallographic_context"]["cell_a_b_c_alpha_beta_gamma"]
+ )
+ ca_rt, cb_rt, cc_rt = runtime_cell[0], runtime_cell[1], runtime_cell[2]
+
+ payload = {
+ "schema_version": 1,
+ "algorithm": "mmtbx.model_vs_data.first_block_public_api",
+ "expected_resolved_prefix": "/opt/phenix",
+ "expected_phenix_version": toolchain.PHENIX_VERSION,
+ "runtime_environment": {
+ "python_version": "3.9.18",
+ "sys_executable": "/opt/phenix/bin/python3.9",
+ "resolved_sys_executable": "/opt/phenix/bin/python3.9",
+ "sys_prefix": "/opt/phenix",
+ "resolved_sys_prefix": "/opt/phenix",
+ "vendor_modules": vendor_modules,
+ },
+ "crystal_symmetry": {
+ "unit_cell": list(runtime_cell),
+ "space_group_symbol": "P 21 21 21 (No. 19)",
+ "space_group_number": 19,
+ "space_group_operators_xyz": [
+ "x,y,z",
+ "-x+1/2,-y,z+1/2",
+ "-x,y+1/2,-z+1/2",
+ "x+1/2,-y+1/2,-z",
+ ],
+ },
+ "extraction": {
+ "labels": ["F-obs", "SIGF-obs"],
+ "r_free_flags_label": "R-free-flags",
+ "test_flag_value": 1,
+ "r_free_flags_generate": False,
+ "ignore_r_free_flags": False,
+ "keep_going": False,
+ "err": [],
+ "log_text": "extract_xtal_data synthetic log\n",
+ "applied_pdb_header_resolution_cut": False,
+ },
+ "diagnostics": {
+ "extract_xtal_data_log": "extract_xtal_data synthetic log\n",
+ "model_manager_log": "model.manager synthetic log\n",
+ "fmodel_update_all_scales_log": "update_all_scales synthetic log\n",
+ "failure": None,
+ },
+ "resolution_ranges": {
+ "extracted_pre_fmodel": {
+ "d_max": cb_rt,
+ "d_min": 1.0
+ / math.sqrt(
+ (1.0 / ca_rt) ** 2
+ + (1.0 / cb_rt) ** 2
+ + (1.0 / cc_rt) ** 2
+ ),
+ },
+ "final_effective": {"d_max": cb_rt, "d_min": ca_rt / 2.0},
+ "actual_work": {"d_max": cb_rt, "d_min": ca_rt},
+ "actual_free": {"d_max": cb_rt / 2.0, "d_min": ca_rt / 2.0},
+ },
+ "xray_structure_invariance": xrs_invariance,
+ "scales_and_rfactors": {
+ "r_work": r_work,
+ "r_free": r_free,
+ "r_all": r_all,
+ "r_free_r_work_gap": r_free - r_work,
+ "scale_k1_w": scale_k1_w,
+ "scale_k1_t": scale_k1_t,
+ "scale_k1": scale_k1,
+ "work_numerator": 20.0,
+ "work_denominator": 300.0,
+ "free_numerator": 50.0,
+ "free_denominator": 200.0,
+ "all_numerator": all_num,
+ "all_denominator": all_den,
+ "work_count": 2,
+ "free_count": 2,
+ "final_effective_count": 4,
+ "extracted_pre_fmodel_count": 5,
+ },
+ "extracted_pre_fmodel": [
+ {"hkl": [1, 0, 0], "f_obs": 100.0, "sigf_obs": 2.0, "is_free": False},
+ {"hkl": [0, 1, 0], "f_obs": 200.0, "sigf_obs": 3.0, "is_free": False},
+ {"hkl": [1, 1, 1], "f_obs": 150.0, "sigf_obs": 2.5, "is_free": False},
+ {"hkl": [2, 0, 0], "f_obs": 80.0, "sigf_obs": 1.8, "is_free": True},
+ {"hkl": [0, 2, 0], "f_obs": 120.0, "sigf_obs": 2.2, "is_free": True},
+ ],
+ "final_effective": [
+ {"hkl": hkl, "f_obs": fo, "sigf_obs": sigfo, "is_free": is_free}
+ for hkl, fo, sigfo, is_free in final_hkls
+ ],
+ "actual_work": [
+ {
+ "hkl": [1, 0, 0],
+ "f_obs": 100.0,
+ "sigf_obs": 2.0,
+ "f_model_real": 108.0,
+ "f_model_imag": 144.0,
+ "f_model_amplitude": 180.0,
+ "f_model_scaled_with_k1_w_real": 54.0,
+ "f_model_scaled_with_k1_w_imag": 72.0,
+ "f_model_scaled_with_k1_w_amplitude": 90.0,
+ },
+ {
+ "hkl": [0, 1, 0],
+ "f_obs": 200.0,
+ "sigf_obs": 3.0,
+ "f_model_real": 252.0,
+ "f_model_imag": 336.0,
+ "f_model_amplitude": 420.0,
+ "f_model_scaled_with_k1_w_real": 126.0,
+ "f_model_scaled_with_k1_w_imag": 168.0,
+ "f_model_scaled_with_k1_w_amplitude": 210.0,
+ },
+ ],
+ "actual_free": [
+ {
+ "hkl": [2, 0, 0],
+ "f_obs": 80.0,
+ "sigf_obs": 1.8,
+ "f_model_real": 60.0,
+ "f_model_imag": 80.0,
+ "f_model_amplitude": 100.0,
+ "f_model_scaled_with_k1_t_real": 36.0,
+ "f_model_scaled_with_k1_t_imag": 48.0,
+ "f_model_scaled_with_k1_t_amplitude": 60.0,
+ },
+ {
+ "hkl": [0, 2, 0],
+ "f_obs": 120.0,
+ "sigf_obs": 2.2,
+ "f_model_real": 150.0,
+ "f_model_imag": 200.0,
+ "f_model_amplitude": 250.0,
+ "f_model_scaled_with_k1_t_real": 90.0,
+ "f_model_scaled_with_k1_t_imag": 120.0,
+ "f_model_scaled_with_k1_t_amplitude": 150.0,
+ },
+ ],
+ "map_coefficients": {
+ "2mFo-DFc": {
+ "map_type": "2mFo-DFc",
+ "amplitude_label": "2FOFCWT",
+ "phase_label": "PH2FOFCWT",
+ "parameters": dict(map_params),
+ "coefficient_semantics": "combine_includes_scales",
+ "n_reflections": 4,
+ "centric_count": 2,
+ "acentric_count": 2,
+ "records": copy.deepcopy(map_records_2fofc),
+ },
+ "mFo-DFc": {
+ "map_type": "mFo-DFc",
+ "amplitude_label": "FOFCWT",
+ "phase_label": "PHFOFCWT",
+ "parameters": dict(map_params),
+ "coefficient_semantics": "combine_includes_scales",
+ "n_reflections": 4,
+ "centric_count": 2,
+ "acentric_count": 2,
+ "records": copy.deepcopy(map_records_fofc),
+ },
+ "mtz_supplements": {
+ "phx_maps_sha256": hashlib.sha256(maps_mtz_bytes).hexdigest(),
+ "phx_maps_columns": [
+ "H",
+ "K",
+ "L",
+ "2FOFCWT",
+ "PH2FOFCWT",
+ "FOFCWT",
+ "PHFOFCWT",
+ ],
+ "obs_pair_only_sha256": hashlib.sha256(obs_pair_mtz_bytes).hexdigest(),
+ "obs_pair_only_columns": [
+ "H",
+ "K",
+ "L",
+ "2FOFCWT",
+ "PH2FOFCWT",
+ ],
+ },
+ },
+ }
+ return input_ledger, payload, vendor_shas, maps_mtz_bytes, obs_pair_mtz_bytes
+
+
+class TestOnesarSubjectAndArchiveAdmission(unittest.TestCase):
+ def test_allowed_subjects_and_forbidden_r7(self) -> None:
+ expected_members = {
+ "start": ("data/1sar.pdb", "start"),
+ "R4": ("data/1sar_final.pdb", "final"),
+ "R5": ("data/1sar_round5.pdb", "intermediate"),
+ "R6": ("data/1sar_round6.pdb", "intermediate"),
+ }
+ for subj, (member, stage) in expected_members.items():
+ spec = onesar_rerun.resolve_subject_spec(subj)
+ self.assertEqual(spec["subject_id"], subj)
+ self.assertEqual(spec["member"], member)
+ self.assertEqual(spec["stage"], stage)
+ self.assertEqual(len(spec["sha256"]), 64)
+
+ for forbidden in (
+ "R7",
+ "round7",
+ "1sar_round7",
+ "data/1sar_round7.pdb",
+ "deposited",
+ "1sar_deposited",
+ "R1",
+ "",
+ ):
+ with self.assertRaises(ValueError):
+ onesar_rerun.resolve_subject_spec(forbidden)
+
+ def test_committed_archive_members_match_preregistered_hashes(self) -> None:
+ self.assertTrue(COMMITTED_ARCHIVE.is_file())
+ for subj in ("start", "R4", "R5", "R6"):
+ spec = onesar_rerun.resolve_subject_spec(subj)
+ data = onesar_rerun.read_verified_archive_member(
+ COMMITTED_ARCHIVE,
+ spec["member"],
+ expected_archive_sha256=onesar_rerun.ARCHIVE_SHA256,
+ expected_member_sha256=spec["sha256"],
+ )
+ self.assertEqual(hashlib.sha256(data).hexdigest(), spec["sha256"])
+
+ mtz_bytes = onesar_rerun.read_verified_archive_member(
+ COMMITTED_ARCHIVE,
+ onesar_rerun.MTZ_MEMBER,
+ expected_archive_sha256=onesar_rerun.ARCHIVE_SHA256,
+ expected_member_sha256=onesar_rerun.MTZ_SHA256,
+ )
+ self.assertEqual(
+ hashlib.sha256(mtz_bytes).hexdigest(), onesar_rerun.MTZ_SHA256
+ )
+
+ def test_archive_member_admission_rejects_bad_hashes_duplicates_symlinks_and_paths(
+ self,
+ ) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ tmp_dir = Path(tmp)
+ zip_path = tmp_dir / "test.zip"
+ payload = b"HEADER TEST\nEND\n"
+ payload_sha = hashlib.sha256(payload).hexdigest()
+
+ with zipfile.ZipFile(zip_path, "w") as zf:
+ zf.writestr("data/model.pdb", payload)
+ zip_sha = hashlib.sha256(zip_path.read_bytes()).hexdigest()
+
+ # Valid read succeeds.
+ self.assertEqual(
+ onesar_rerun.read_verified_archive_member(
+ zip_path,
+ "data/model.pdb",
+ expected_archive_sha256=zip_sha,
+ expected_member_sha256=payload_sha,
+ ),
+ payload,
+ )
+
+ # Wrong archive hash fails.
+ with self.assertRaisesRegex(ValueError, "Archive SHA-256 mismatch"):
+ onesar_rerun.read_verified_archive_member(
+ zip_path,
+ "data/model.pdb",
+ expected_archive_sha256="0" * 64,
+ expected_member_sha256=payload_sha,
+ )
+
+ # Wrong member hash fails.
+ with self.assertRaisesRegex(ValueError, "SHA-256 mismatch"):
+ onesar_rerun.read_verified_archive_member(
+ zip_path,
+ "data/model.pdb",
+ expected_archive_sha256=zip_sha,
+ expected_member_sha256="f" * 64,
+ )
+
+ # Unsafe member names fail before touching the archive.
+ for bad_name in ("/etc/passwd", "../secret.pdb", "data\\model.pdb", "data//model.pdb"):
+ with self.assertRaisesRegex(ValueError, "Unsafe archive member"):
+ onesar_rerun.read_verified_archive_member(
+ zip_path,
+ bad_name,
+ expected_archive_sha256=zip_sha,
+ expected_member_sha256=payload_sha,
+ )
+
+ # Symlink to archive fails.
+ link_zip = tmp_dir / "link.zip"
+ link_zip.symlink_to(zip_path)
+ with self.assertRaisesRegex(ValueError, "Regular non-symlink file"):
+ onesar_rerun.read_verified_archive_member(
+ link_zip,
+ "data/model.pdb",
+ expected_archive_sha256=zip_sha,
+ expected_member_sha256=payload_sha,
+ )
+
+ # Duplicate member entries fail (suppress zipfile's own UserWarning on write).
+ dup_zip = tmp_dir / "dup.zip"
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", UserWarning)
+ with zipfile.ZipFile(dup_zip, "w") as zf:
+ zf.writestr("data/model.pdb", payload)
+ zf.writestr("data/model.pdb", payload)
+ dup_sha = hashlib.sha256(dup_zip.read_bytes()).hexdigest()
+ with self.assertRaisesRegex(ValueError, "duplicate entries"):
+ onesar_rerun.read_verified_archive_member(
+ dup_zip,
+ "data/model.pdb",
+ expected_archive_sha256=dup_sha,
+ expected_member_sha256=payload_sha,
+ )
+
+ # Symlink ZIP member entry fails.
+ sym_zip = tmp_dir / "sym_member.zip"
+ info = zipfile.ZipInfo("data/model.pdb")
+ info.external_attr = (0o120777) << 16
+ with zipfile.ZipFile(sym_zip, "w") as zf:
+ zf.writestr(info, payload)
+ sym_sha = hashlib.sha256(sym_zip.read_bytes()).hexdigest()
+ with self.assertRaisesRegex(ValueError, "is a symlink"):
+ onesar_rerun.read_verified_archive_member(
+ sym_zip,
+ "data/model.pdb",
+ expected_archive_sha256=sym_sha,
+ expected_member_sha256=payload_sha,
+ )
+
+
+class TestFreshEvidenceRootBoundary(unittest.TestCase):
+ def test_fresh_evidence_root_rejects_existing_symlinks_and_outside_repo(
+ self,
+ ) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ fake_repo = Path(tmp).resolve(strict=True) / "repo"
+ fake_repo.mkdir()
+
+ fresh = onesar_rerun.prepare_fresh_evidence_root(
+ fake_repo / "runs" / "canary_01", repo_root=fake_repo
+ )
+ self.assertTrue(fresh.is_dir())
+
+ # Reusing an existing directory must fail with FileExistsError.
+ with self.assertRaises(FileExistsError):
+ onesar_rerun.prepare_fresh_evidence_root(fresh, repo_root=fake_repo)
+
+ # Repo root itself is rejected.
+ with self.assertRaisesRegex(ValueError, "strict subdirectory"):
+ onesar_rerun.prepare_fresh_evidence_root(fake_repo, repo_root=fake_repo)
+
+ # Path outside repo root is rejected.
+ outside = Path(tmp).resolve(strict=True) / "outside_dir"
+ with self.assertRaisesRegex(ValueError, "strict subdirectory"):
+ onesar_rerun.prepare_fresh_evidence_root(outside, repo_root=fake_repo)
+
+ # Symlink ancestor inside repo is rejected.
+ real_sub = fake_repo / "real_sub"
+ real_sub.mkdir()
+ sym_sub = fake_repo / "sym_sub"
+ sym_sub.symlink_to(real_sub)
+ with self.assertRaisesRegex(ValueError, "must not be a symlink"):
+ onesar_rerun.prepare_fresh_evidence_root(
+ sym_sub / "child", repo_root=fake_repo
+ )
+
+
+class TestMtzProjectionAndHklLedger(unittest.TestCase):
+ def test_committed_1sar_mtz_projection_and_ledger_counts(self) -> None:
+ mtz_bytes = onesar_rerun.read_verified_archive_member(
+ COMMITTED_ARCHIVE,
+ onesar_rerun.MTZ_MEMBER,
+ expected_archive_sha256=onesar_rerun.ARCHIVE_SHA256,
+ expected_member_sha256=onesar_rerun.MTZ_SHA256,
+ )
+ projected_bytes, proj_summary = onesar_rerun.project_observation_mtz(mtz_bytes)
+ self.assertEqual(proj_summary["input_ncols"], 16)
+ self.assertEqual(proj_summary["projected_ncols"], 6)
+ self.assertEqual(proj_summary["nrows"], 7248)
+ self.assertEqual(
+ proj_summary["projected_columns"],
+ ["H", "K", "L", "F-obs", "SIGF-obs", "R-free-flags"],
+ )
+ self.assertEqual(proj_summary["retained_dataset_ids"], ["1"])
+ self.assertEqual(
+ set(proj_summary["crystallographic_context"]),
+ {
+ "cell_a_b_c_alpha_beta_gamma",
+ "resolution_limits_inverse_a2",
+ "symmetry",
+ },
+ )
+ self.assertEqual(
+ proj_summary["retention_audit"]["audit_status"], "preserved"
+ )
+
+ ledger = onesar_rerun.build_input_hkl_ledger(projected_bytes)
+ self.assertEqual(
+ ledger["crystallographic_context"],
+ proj_summary["crystallographic_context"],
+ )
+ counts = ledger["counts"]
+ self.assertEqual(counts["raw_total"], 7248)
+ self.assertEqual(counts["raw_work"], 6505)
+ self.assertEqual(counts["raw_free"], 723)
+ self.assertEqual(counts["raw_unassigned"], 20)
+ self.assertEqual(counts["eligible_total"], 7228)
+ self.assertEqual(counts["eligible_work"], 6505)
+ self.assertEqual(counts["eligible_free"], 723)
+ self.assertEqual(counts["eligible_zero_f"], 11)
+ self.assertEqual(counts["eligible_negative_f"], 0)
+ self.assertEqual(counts["ineligible_total"], 20)
+ self.assertEqual(counts["ineligible_nonfinite_f"], 20)
+ self.assertEqual(counts["ineligible_nonpositive_or_nonfinite_sigf"], 0)
+ self.assertEqual(counts["ineligible_unassigned_flag"], 20)
+ self.assertEqual(len(ledger["ineligible_rows"]), 20)
+ for row in ledger["ineligible_rows"]:
+ self.assertIsNone(row["f_obs"])
+ self.assertIsNone(row["sigf_obs"])
+ self.assertIsNone(row["r_free_flag"])
+ self.assertIsNone(row["is_free"])
+ self.assertEqual(row["partition"], "unassigned")
+ self.assertEqual(len(row["f_obs_word_hex"]), 8)
+ self.assertEqual(len(row["sigf_obs_word_hex"]), 8)
+ self.assertEqual(len(row["r_free_flag_word_hex"]), 8)
+
+ def test_synthetic_mtz_eligibility_zero_negative_nan_and_failure_modes(
+ self,
+ ) -> None:
+ nan = float("nan")
+ # Include archived columns in synthetic MTZ to verify projection strips them.
+ cols = [
+ ("H", "H", "1"),
+ ("K", "H", "1"),
+ ("L", "H", "1"),
+ ("F-obs", "F", "1"),
+ ("SIGF-obs", "Q", "1"),
+ ("R-free-flags", "I", "1"),
+ ("F-model", "F", "2"),
+ ("2FOFCWT", "F", "3"),
+ ]
+ datasets = [
+ (1, "Original-experimental-data-mapped-to-asu"),
+ (2, "Model-structure-factors"),
+ (3, "Electron-density"),
+ ]
+ mtz_bytes = _build_synthetic_mtz(
+ [
+ (1, 0, 0, 50.0, 2.0, 0.0, 48.0, 51.0),
+ (0, 1, 0, 0.0, 1.0, 0.0, 1.0, 0.5),
+ (0, 0, 1, -3.5, 0.8, 1.0, 2.0, 0.2),
+ (1, 1, 0, nan, nan, 0.0, 10.0, 10.0),
+ (1, 0, 1, 25.0, 0.0, 1.0, 20.0, 22.0),
+ (0, 1, 1, 30.0, -1.0, 0.0, 28.0, 29.0),
+ (1, 1, 1, nan, nan, nan, nan, nan),
+ ],
+ columns=cols,
+ datasets=datasets,
+ )
+ proj_bytes, proj_summary = onesar_rerun.project_observation_mtz(mtz_bytes)
+ self.assertEqual(proj_summary["input_ncols"], 8)
+ self.assertEqual(proj_summary["projected_ncols"], 6)
+ self.assertEqual(proj_summary["retained_dataset_ids"], ["1"])
+
+ ledger = onesar_rerun.build_input_hkl_ledger(proj_bytes)
+ counts = ledger["counts"]
+ self.assertEqual(counts["raw_total"], 7)
+ self.assertEqual(counts["raw_work"], 4)
+ self.assertEqual(counts["raw_free"], 2)
+ self.assertEqual(counts["raw_unassigned"], 1)
+ self.assertEqual(counts["eligible_total"], 3)
+ self.assertEqual(counts["eligible_work"], 2)
+ self.assertEqual(counts["eligible_free"], 1)
+ self.assertEqual(counts["eligible_zero_f"], 1)
+ self.assertEqual(counts["eligible_negative_f"], 1)
+ self.assertEqual(counts["ineligible_total"], 4)
+ self.assertEqual(counts["ineligible_nonfinite_f"], 2)
+ self.assertEqual(counts["ineligible_nonpositive_or_nonfinite_sigf"], 2)
+ self.assertEqual(counts["ineligible_unassigned_flag"], 1)
+ self.assertTrue(ledger["eligible_by_hkl"]["0,1,0"]["is_zero_f"])
+ self.assertTrue(ledger["eligible_by_hkl"]["0,0,1"]["is_negative_f"])
+ unassigned_rows = [
+ r for r in ledger["ineligible_rows"] if r["hkl"] == [1, 1, 1]
+ ]
+ self.assertEqual(len(unassigned_rows), 1)
+ self.assertIsNone(unassigned_rows[0]["r_free_flag"])
+ self.assertIsNone(unassigned_rows[0]["is_free"])
+ self.assertEqual(unassigned_rows[0]["partition"], "unassigned")
+
+ # Duplicate HKL fails.
+ dup_mtz = _build_synthetic_mtz(
+ [
+ (1, 0, 0, 50.0, 2.0, 0.0),
+ (1, 0, 0, 60.0, 2.5, 1.0),
+ ]
+ )
+ with self.assertRaisesRegex(ValueError, "Duplicate Miller index"):
+ onesar_rerun.build_input_hkl_ledger(dup_mtz)
+
+ # NaN R-free flag on an eligible reflection fails.
+ nan_flag_mtz = _build_synthetic_mtz(
+ [
+ (1, 0, 0, 50.0, 2.0, nan),
+ (0, 1, 0, 60.0, 2.5, 1.0),
+ ]
+ )
+ with self.assertRaisesRegex(ValueError, "Invalid R-free flag value"):
+ onesar_rerun.build_input_hkl_ledger(nan_flag_mtz)
+
+ # Undeclared integer flag (e.g. 2) on an eligible or ineligible reflection fails.
+ bad_flag_mtz = _build_synthetic_mtz(
+ [
+ (1, 0, 0, 50.0, 2.0, 0.0),
+ (0, 1, 0, 60.0, 2.5, 2.0),
+ ]
+ )
+ with self.assertRaisesRegex(ValueError, "Invalid R-free flag value"):
+ onesar_rerun.build_input_hkl_ledger(bad_flag_mtz)
+
+ bad_ineligible_flag_mtz = _build_synthetic_mtz(
+ [
+ (1, 0, 0, 50.0, 2.0, 0.0),
+ (0, 1, 0, 60.0, 2.5, 1.0),
+ (0, 0, 1, nan, nan, 2.0),
+ ]
+ )
+ with self.assertRaisesRegex(ValueError, "Invalid R-free flag value"):
+ onesar_rerun.build_input_hkl_ledger(bad_ineligible_flag_mtz)
+
+ # Non-integral Miller index fails.
+ frac_hkl_mtz = _build_synthetic_mtz(
+ [
+ (1.5, 0, 0, 50.0, 2.0, 0.0),
+ (0, 1, 0, 60.0, 2.5, 1.0),
+ ]
+ )
+ with self.assertRaisesRegex(ValueError, "Non-integral Miller index"):
+ onesar_rerun.build_input_hkl_ledger(frac_hkl_mtz)
+
+
+class TestPhenixReconciliationAndPureHelpers(unittest.TestCase):
+ def _reconcile(
+ self,
+ payload: dict[str, Any],
+ ledger: dict[str, Any],
+ vendor_shas: dict[str, str],
+ maps_bytes: bytes,
+ pair_bytes: bytes,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ return onesar_rerun.reconcile_phenix_rfactor_output(
+ payload,
+ ledger,
+ expected_vendor_source_sha256=vendor_shas,
+ phx_maps_mtz_bytes=maps_bytes,
+ phx_obs_pair_only_mtz_bytes=pair_bytes,
+ **kwargs,
+ )
+
+ def test_reconcile_valid_payload_and_stage_exclusions(self) -> None:
+ ledger, payload, vendor_shas, maps_bytes, pair_bytes = (
+ _make_reconciliation_fixture()
+ )
+ result = self._reconcile(
+ payload, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+ self.assertEqual(result["reconciliation_status"], "verified")
+ self.assertEqual(result["counts"]["raw_mtz_total"], 8)
+ self.assertEqual(result["counts"]["input_eligible_total"], 7)
+ self.assertEqual(result["counts"]["extracted_pre_fmodel_total"], 5)
+ self.assertEqual(result["counts"]["final_effective_total"], 4)
+ self.assertEqual(result["counts"]["actual_work_total"], 2)
+ self.assertEqual(result["counts"]["actual_free_total"], 2)
+ self.assertEqual(
+ result["set_differences"]["raw_to_input_eligible_excluded_hkls"],
+ [[0, 0, 2]],
+ )
+ self.assertEqual(
+ result["set_differences"]["input_eligible_to_extracted_excluded_hkls"],
+ [[0, 0, 1], [1, 1, 0]],
+ )
+ self.assertEqual(
+ result["set_differences"]["extracted_to_final_effective_excluded_hkls"],
+ [[1, 1, 1]],
+ )
+ self.assertAlmostEqual(
+ result["rfactors"]["replayed_r_work"], 20.0 / 300.0, places=15
+ )
+ self.assertAlmostEqual(
+ result["rfactors"]["replayed_r_free"], 0.25, places=15
+ )
+ self.assertAlmostEqual(
+ result["rfactors"]["replayed_r_all"], 70.6 / 500.0, places=15
+ )
+ self.assertEqual(
+ set(result["resolution_ranges"]),
+ {
+ "extracted_pre_fmodel",
+ "final_effective",
+ "actual_work",
+ "actual_free",
+ },
+ )
+ ca_f32, cb_f32 = (
+ struct.unpack(" None:
+ ledger, payload, vendor_shas, maps_bytes, pair_bytes = (
+ _make_reconciliation_fixture()
+ )
+ # Substitute work scale_k1_w (0.5) in place of free scale_k1_t (0.6) on free set.
+ bad = copy.deepcopy(payload)
+ bad["actual_free"][0]["f_model_scaled_with_k1_t_real"] = 30.0
+ bad["actual_free"][0]["f_model_scaled_with_k1_t_imag"] = 40.0
+ bad["actual_free"][0]["f_model_scaled_with_k1_t_amplitude"] = 50.0
+ with self.assertRaisesRegex(
+ ValueError, "disagrees with scale_k1_t \\* \\|f_model_free\\|"
+ ):
+ self._reconcile(bad, ledger, vendor_shas, maps_bytes, pair_bytes)
+
+ # Complex sign reversal or imaginary sign flip that preserves amplitude and
+ # scalar R_work / R_free must be rejected for both actual_work and actual_free (#1021).
+ work_sign_flip = copy.deepcopy(payload)
+ work_sign_flip["actual_work"][0]["f_model_scaled_with_k1_w_real"] = (
+ -54.0
+ )
+ work_sign_flip["actual_work"][0]["f_model_scaled_with_k1_w_imag"] = (
+ -72.0
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "actual_work\\[1,0,0\\] f_model_scaled_with_k1_w complex components .* disagree with scale_k1_w \\* f_model_work",
+ ):
+ self._reconcile(
+ work_sign_flip, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ work_imag_flip = copy.deepcopy(payload)
+ work_imag_flip["actual_work"][1]["f_model_scaled_with_k1_w_imag"] = (
+ -168.0
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "actual_work\\[0,1,0\\] f_model_scaled_with_k1_w complex components .* disagree with scale_k1_w \\* f_model_work",
+ ):
+ self._reconcile(
+ work_imag_flip, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ free_sign_flip = copy.deepcopy(payload)
+ free_sign_flip["actual_free"][0]["f_model_scaled_with_k1_t_real"] = (
+ -36.0
+ )
+ free_sign_flip["actual_free"][0]["f_model_scaled_with_k1_t_imag"] = (
+ -48.0
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "actual_free\\[2,0,0\\] f_model_scaled_with_k1_t complex components .* disagree with scale_k1_t \\* f_model_free",
+ ):
+ self._reconcile(
+ free_sign_flip, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ free_imag_flip = copy.deepcopy(payload)
+ free_imag_flip["actual_free"][1]["f_model_scaled_with_k1_t_imag"] = (
+ -120.0
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "actual_free\\[0,2,0\\] f_model_scaled_with_k1_t complex components .* disagree with scale_k1_t \\* f_model_free",
+ ):
+ self._reconcile(
+ free_imag_flip, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ def test_reconcile_rejects_partition_flag_and_set_violations(self) -> None:
+ ledger, payload, vendor_shas, maps_bytes, pair_bytes = (
+ _make_reconciliation_fixture()
+ )
+
+ # Overlapping work and free HKL fails.
+ overlap = copy.deepcopy(payload)
+ overlap["actual_free"].append(
+ {
+ "hkl": [1, 0, 0],
+ "f_obs": 100.0,
+ "sigf_obs": 2.0,
+ "f_model_real": 100.0,
+ "f_model_imag": 0.0,
+ "f_model_amplitude": 100.0,
+ "f_model_scaled_with_k1_t_real": 60.0,
+ "f_model_scaled_with_k1_t_imag": 0.0,
+ "f_model_scaled_with_k1_t_amplitude": 60.0,
+ }
+ )
+ with self.assertRaisesRegex(ValueError, "non-disjoint"):
+ self._reconcile(overlap, ledger, vendor_shas, maps_bytes, pair_bytes)
+
+ # Swapped 0/1 flag assignment fails.
+ swapped = copy.deepcopy(payload)
+ swapped["extracted_pre_fmodel"][0]["is_free"] = True
+ with self.assertRaisesRegex(ValueError, "Flag mismatch at extracted HKL"):
+ self._reconcile(swapped, ledger, vendor_shas, maps_bytes, pair_bytes)
+
+ # Unknown HKL not in raw MTZ fails.
+ unknown = copy.deepcopy(payload)
+ unknown["extracted_pre_fmodel"].append(
+ {"hkl": [9, 9, 9], "f_obs": 50.0, "sigf_obs": 1.0, "is_free": False}
+ )
+ with self.assertRaisesRegex(ValueError, "absent from input MTZ raw ledger"):
+ self._reconcile(unknown, ledger, vendor_shas, maps_bytes, pair_bytes)
+
+ # Second-block PDB header cutoff flag fails.
+ second_block = copy.deepcopy(payload)
+ second_block["extraction"]["applied_pdb_header_resolution_cut"] = True
+ with self.assertRaisesRegex(
+ ValueError, "Second-block PDB header resolution cutoff"
+ ):
+ self._reconcile(
+ second_block, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ def test_reconcile_rejects_scatterer_mutation_map_contract_and_prefix_mix(
+ self,
+ ) -> None:
+ ledger, payload, vendor_shas, maps_bytes, pair_bytes = (
+ _make_reconciliation_fixture()
+ )
+
+ # Scatterer coordinate or occupancy drift fails even when top-level diff is spoofed.
+ mut_xrs = copy.deepcopy(payload)
+ mut_xrs["xray_structure_invariance"]["max_abs_occupancy_diff"] = 1e-4
+ with self.assertRaisesRegex(
+ ValueError, "Non-zero max_abs_occupancy_diff"
+ ):
+ self._reconcile(mut_xrs, ledger, vendor_shas, maps_bytes, pair_bytes)
+
+ # Mutate a scatterer record in after_fmodel without updating its sha256 -> fails digest check.
+ mut_snap = copy.deepcopy(payload)
+ mut_snap["xray_structure_invariance"]["snapshots"]["after_fmodel"][
+ "records"
+ ][0]["site_cart"][0] += 0.01
+ with self.assertRaisesRegex(
+ ValueError, "after_fmodel snapshot sha256 mismatch"
+ ):
+ self._reconcile(mut_snap, ledger, vendor_shas, maps_bytes, pair_bytes)
+
+ # Mutate a scatterer record AND recompute after_fmodel["sha256"] while spoofing
+ # top-level after_fmodel_sha256 and max_abs_site_cart_diff=0 -> still caught!
+ spoofed_snap = copy.deepcopy(payload)
+ bad_records = copy.deepcopy(
+ spoofed_snap["xray_structure_invariance"]["snapshots"]["after_fmodel"][
+ "records"
+ ]
+ )
+ bad_records[0]["occupancy"] = 0.75
+ recomputed_bad = onesar_phenix_measure.snapshot_scatterer_records(
+ bad_records
+ )
+ spoofed_snap["xray_structure_invariance"]["snapshots"]["after_fmodel"] = (
+ recomputed_bad
+ )
+ with self.assertRaisesRegex(ValueError, "Scatterer snapshot mutated"):
+ self._reconcile(
+ spoofed_snap, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # Missing snapshots fails.
+ no_snaps = copy.deepcopy(payload)
+ del no_snaps["xray_structure_invariance"]["snapshots"]
+ with self.assertRaisesRegex(
+ ValueError,
+ "xray_structure_invariance must contain snapshots",
+ ):
+ self._reconcile(no_snaps, ledger, vendor_shas, maps_bytes, pair_bytes)
+
+ # Map coefficient fill_missing=True fails.
+ bad_map = copy.deepcopy(payload)
+ bad_map["map_coefficients"]["2mFo-DFc"]["parameters"]["fill_missing"] = True
+ with self.assertRaisesRegex(
+ ValueError, "parameters violate preregistered contract"
+ ):
+ self._reconcile(bad_map, ledger, vendor_shas, maps_bytes, pair_bytes)
+
+ # Mixed vendor sys.prefix fails.
+ bad_prefix = copy.deepcopy(payload)
+ bad_prefix["runtime_environment"]["vendor_modules"][
+ "vendor_mmtbx_model_vs_data"
+ ]["resolved_path"] = "/other/install/mmtbx/model_vs_data.py"
+ with self.assertRaisesRegex(ValueError, "escapes resolved_sys_prefix"):
+ self._reconcile(
+ bad_prefix, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # Same source hashes under a different installation prefix fails (#1013).
+ other_prefix = copy.deepcopy(payload)
+ other_prefix["expected_resolved_prefix"] = "/other/phenix"
+ other_prefix["runtime_environment"]["sys_prefix"] = "/other/phenix"
+ other_prefix["runtime_environment"]["resolved_sys_prefix"] = "/other/phenix"
+ other_prefix["runtime_environment"]["sys_executable"] = (
+ "/other/phenix/bin/python3.9"
+ )
+ other_prefix["runtime_environment"]["resolved_sys_executable"] = (
+ "/other/phenix/bin/python3.9"
+ )
+ for mod_key, rel in onesar_rerun.PHENIX_VENDOR_SOURCE_RELPATHS.items():
+ other_prefix["runtime_environment"]["vendor_modules"][mod_key][
+ "resolved_path"
+ ] = f"/other/phenix/lib/python3.9/site-packages/{rel}"
+ with self.assertRaisesRegex(
+ ValueError, "resolved_sys_prefix .* != expected_resolved_prefix"
+ ):
+ self._reconcile(
+ other_prefix,
+ ledger,
+ vendor_shas,
+ maps_bytes,
+ pair_bytes,
+ expected_resolved_prefix="/opt/phenix",
+ )
+
+ # Wrong PHENIX version in payload fails (#1013).
+ bad_ver = copy.deepcopy(payload)
+ bad_ver["expected_phenix_version"] = "2.0-5935"
+ with self.assertRaisesRegex(
+ ValueError, "expected_phenix_version '2.0-5935' !="
+ ):
+ self._reconcile(bad_ver, ledger, vendor_shas, maps_bytes, pair_bytes)
+
+ def test_full_precision_counts_sums_and_r_all_reconciliation_1015(self) -> None:
+ ledger, payload, vendor_shas, maps_bytes, pair_bytes = (
+ _make_reconciliation_fixture()
+ )
+
+ # Per-HKL f_obs mismatch between final_effective and actual_work fails.
+ bad_final_fo = copy.deepcopy(payload)
+ bad_final_fo["final_effective"][0]["f_obs"] = 100.5
+ with self.assertRaisesRegex(
+ ValueError, "actual_work\\[1,0,0\\] F/SIGF .* disagrees with final_effective"
+ ):
+ self._reconcile(
+ bad_final_fo, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # Per-HKL sigf_obs mismatch between final_effective and actual_free fails.
+ bad_final_sigf = copy.deepcopy(payload)
+ bad_final_sigf["final_effective"][2]["sigf_obs"] = 9.9
+ with self.assertRaisesRegex(
+ ValueError, "actual_free\\[2,0,0\\] F/SIGF .* disagrees with final_effective"
+ ):
+ self._reconcile(
+ bad_final_sigf, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # Mutated work_count / free_count / final_effective_count / extracted_pre_fmodel_count fails.
+ for count_key in (
+ "work_count",
+ "free_count",
+ "final_effective_count",
+ "extracted_pre_fmodel_count",
+ ):
+ bad_cnt = copy.deepcopy(payload)
+ bad_cnt["scales_and_rfactors"][count_key] += 1
+ with self.assertRaisesRegex(
+ ValueError, "Reported scales_and_rfactors counts disagree"
+ ):
+ self._reconcile(
+ bad_cnt, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+ bool_cnt = copy.deepcopy(payload)
+ bool_cnt["scales_and_rfactors"][count_key] = True
+ with self.assertRaisesRegex(ValueError, "must be a positive int"):
+ self._reconcile(
+ bool_cnt, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # Mutated numerator/denominator sums fail.
+ for sum_key, err_pat in (
+ ("work_numerator", "Reported work_numerator"),
+ ("work_denominator", "Reported work_denominator"),
+ ("free_numerator", "Reported free_numerator"),
+ ("free_denominator", "Reported free_denominator"),
+ ("all_numerator", "Reported all_numerator"),
+ ("all_denominator", "Reported all_denominator"),
+ ):
+ bad_sum = copy.deepcopy(payload)
+ bad_sum["scales_and_rfactors"][sum_key] += 1.0
+ with self.assertRaisesRegex(ValueError, err_pat):
+ self._reconcile(
+ bad_sum, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # Mutated r_all (e.g. 0.14 instead of 70.6 / 500.0 = 0.1412) fails.
+ bad_r_all = copy.deepcopy(payload)
+ bad_r_all["scales_and_rfactors"]["r_all"] = 0.14
+ with self.assertRaisesRegex(
+ ValueError, "Replayed r_all .* disagrees with native fmodel.r_all"
+ ):
+ self._reconcile(
+ bad_r_all, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ def test_resolution_ranges_reconciliation_and_cell_metric_1020(
+ self,
+ ) -> None:
+ ledger, payload, vendor_shas, maps_bytes, pair_bytes = (
+ _make_reconciliation_fixture()
+ )
+
+ # 0. Actual parser-produced ledger regression (#1008 / #1020):
+ # mtz_crystallographic_context emits exact decimal string tokens (e.g. '64.897', '9E+1').
+ parser_tokens = ledger["crystallographic_context"][
+ "cell_a_b_c_alpha_beta_gamma"
+ ]
+ self.assertEqual(
+ parser_tokens,
+ ["64.897", "78.323", "38.792", "9E+1", "9E+1", "9E+1"],
+ )
+ self.assertTrue(all(type(tok) is str for tok in parser_tokens))
+ self.assertEqual(
+ onesar_rerun.validate_unit_cell_parameters(parser_tokens),
+ (64.897, 78.323, 38.792, 90.0, 90.0, 90.0),
+ )
+ expected_f32_cell = struct.unpack(
+ "<6f", struct.pack("<6f", 64.897, 78.323, 38.792, 90.0, 90.0, 90.0)
+ )
+ self.assertEqual(
+ onesar_rerun.ccp4_mtz_float32_unit_cell(parser_tokens),
+ expected_f32_cell,
+ )
+ baseline_res = self._reconcile(
+ payload, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+ self.assertEqual(baseline_res["reconciliation_status"], "verified")
+ self.assertEqual(
+ baseline_res["crystal_symmetry"]["input_header_cell_tokens"],
+ ["64.897", "78.323", "38.792", "9E+1", "9E+1", "9E+1"],
+ )
+ self.assertEqual(
+ baseline_res["crystal_symmetry"]["input_header_cell_float64"],
+ [64.897, 78.323, 38.792, 90.0, 90.0, 90.0],
+ )
+ self.assertEqual(
+ baseline_res["crystal_symmetry"]["effective_runtime_unit_cell"],
+ list(expected_f32_cell),
+ )
+ self.assertEqual(
+ baseline_res["crystal_symmetry"]["max_abs_runtime_vs_float32_diff"],
+ 0.0,
+ )
+ self.assertEqual(
+ baseline_res["crystal_symmetry"]["space_group_symbol"],
+ "P 21 21 21 (No. 19)",
+ )
+ self.assertEqual(
+ baseline_res["crystal_symmetry"]["space_group_number"], 19
+ )
+ self.assertEqual(
+ baseline_res["crystal_symmetry"]["raw_space_group_operators_xyz"],
+ [
+ "x,y,z",
+ "-x+1/2,-y,z+1/2",
+ "-x,y+1/2,-z+1/2",
+ "x+1/2,-y+1/2,-z",
+ ],
+ )
+ self.assertEqual(
+ baseline_res["crystal_symmetry"][
+ "normalized_space_group_operators_xyz"
+ ],
+ [
+ "-X+1/2,-Y,Z+1/2",
+ "-X,Y+1/2,-Z+1/2",
+ "X+1/2,-Y+1/2,-Z",
+ "X,Y,Z",
+ ],
+ )
+ self.assertEqual(
+ baseline_res["crystal_symmetry"]["normalized_lookup_symbol"],
+ "P212121",
+ )
+ self.assertEqual(
+ baseline_res["crystal_symmetry"]["symmetry_normalization_policy"],
+ onesar_rerun.SYMMETRY_OPERATOR_NORMALIZATION_POLICY,
+ )
+ # Reconciliation must preserve the original string tokens in the ledger untouched.
+ self.assertEqual(
+ ledger["crystallographic_context"]["cell_a_b_c_alpha_beta_gamma"],
+ ["64.897", "78.323", "38.792", "9E+1", "9E+1", "9E+1"],
+ )
+ self.assertTrue(
+ all(
+ type(tok) is str
+ for tok in ledger["crystallographic_context"][
+ "cell_a_b_c_alpha_beta_gamma"
+ ]
+ )
+ )
+
+ # Unrounded decimal-header resolution ranges or unrounded runtime unit_cell are rejected:
+ unrounded_ranges_payload = copy.deepcopy(payload)
+ unrounded_ranges_payload["resolution_ranges"]["extracted_pre_fmodel"][
+ "d_max"
+ ] = 78.323
+ with self.assertRaisesRegex(
+ ValueError,
+ "resolution_ranges\\['extracted_pre_fmodel'\\]\\.d_max 78\\.322999999999993 disagrees with recomputed 78\\.322998046875000",
+ ):
+ self._reconcile(
+ unrounded_ranges_payload,
+ ledger,
+ vendor_shas,
+ maps_bytes,
+ pair_bytes,
+ )
+
+ unrounded_runtime_cell_payload = copy.deepcopy(payload)
+ unrounded_runtime_cell_payload["crystal_symmetry"]["unit_cell"] = [
+ 64.897,
+ 78.323,
+ 38.792,
+ 90.0,
+ 90.0,
+ 90.0,
+ ]
+ with self.assertRaisesRegex(
+ ValueError,
+ "phenix_payload\\.crystal_symmetry\\.unit_cell\\[0\\] \\(a\\) .* disagrees with installed CCP4/iotbx float32 conversion",
+ ):
+ self._reconcile(
+ unrounded_runtime_cell_payload,
+ ledger,
+ vendor_shas,
+ maps_bytes,
+ pair_bytes,
+ )
+
+ for idx_symm, (bad_symm_fn, symm_err) in enumerate((
+ (
+ lambda p: p.pop("crystal_symmetry"),
+ "phenix_payload missing crystal_symmetry section",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_symbol": ""}
+ ),
+ "space_group_symbol must be a non-empty string",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"unit_cell": [64.897, 78.323, 38.792]}
+ ),
+ "unit_cell must be a 6-element numeric sequence",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {
+ "unit_cell": [
+ "64.89700317382812",
+ 78.322998046875,
+ 38.79199981689453,
+ 90.0,
+ 90.0,
+ 90.0,
+ ]
+ }
+ ),
+ "must be a finite number",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {
+ "unit_cell": [
+ 65.0,
+ 78.322998046875,
+ 38.79199981689453,
+ 90.0,
+ 90.0,
+ 90.0,
+ ]
+ }
+ ),
+ "disagrees with installed CCP4/iotbx float32 conversion",
+ ),
+ # Missing / invalid / inconsistent runtime space_group_number (#1025):
+ (
+ lambda p: p["crystal_symmetry"].pop("space_group_number"),
+ "space_group_number must be an int in 1..230",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_number": None}
+ ),
+ "space_group_number must be an int in 1..230",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_number": True}
+ ),
+ "space_group_number must be an int in 1..230",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_number": "19"}
+ ),
+ "space_group_number must be an int in 1..230",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_number": 0}
+ ),
+ "space_group_number must be an int in 1..230",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_number": 231}
+ ),
+ "space_group_number must be an int in 1..230",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_number": 1}
+ ),
+ "space_group_number 1 disagrees with input_ledger space_group_number 19",
+ ),
+ # Missing / invalid / unsupported-spelling / inconsistent runtime space_group_operators_xyz (#1025):
+ (
+ lambda p: p["crystal_symmetry"].pop(
+ "space_group_operators_xyz"
+ ),
+ "space_group_operators_xyz must be a non-empty sequence",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_operators_xyz": None}
+ ),
+ "space_group_operators_xyz must be a non-empty sequence",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_operators_xyz": []}
+ ),
+ "space_group_operators_xyz must be a non-empty sequence",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_operators_xyz": ["x,y,z"]}
+ ),
+ "space_group_operators_xyz .* disagrees with input_ledger symmetry\\.operators_xyz",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {
+ "space_group_operators_xyz": [
+ "x,y,z",
+ "X,Y,Z",
+ "-x,y+1/2,-z+1/2",
+ "x+1/2,-y+1/2,-z",
+ ]
+ }
+ ),
+ "contains duplicate normalized symmetry operators",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {
+ "space_group_operators_xyz": [
+ "x,y,z",
+ "-x+1/4,-y,z+1/2",
+ "-x,y+1/2,-z+1/2",
+ "x+1/2,-y+1/2,-z",
+ ]
+ }
+ ),
+ "space_group_operators_xyz .* disagrees with input_ledger symmetry\\.operators_xyz",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {
+ "space_group_operators_xyz": [
+ "x,y,z",
+ "-x+0.5,-y,z+0.5",
+ "-x,y+1/2,-z+1/2",
+ "x+1/2,-y+1/2,-z",
+ ]
+ }
+ ),
+ "uses unsupported or malformed symmetry operator spelling",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {
+ "space_group_operators_xyz": [
+ "x,y,z",
+ "1/2-x,-y,1/2+z",
+ "-x,y+1/2,-z+1/2",
+ "x+1/2,-y+1/2,-z",
+ ]
+ }
+ ),
+ "uses unsupported or malformed symmetry operator spelling",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {
+ "space_group_operators_xyz": [
+ "x,y,z",
+ "-x+2/4,-y,z+1/2",
+ "-x,y+1/2,-z+1/2",
+ "x+1/2,-y+1/2,-z",
+ ]
+ }
+ ),
+ "has non-canonical or unreduced translation",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {
+ "space_group_operators_xyz": [
+ "+x,y,z",
+ "-x+1/2,-y,z+1/2",
+ "-x,y+1/2,-z+1/2",
+ "x+1/2,-y+1/2,-z",
+ ]
+ }
+ ),
+ "uses unsupported or malformed symmetry operator spelling",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {
+ "space_group_operators_xyz": [
+ "a,b,c",
+ "-x+1/2,-y,z+1/2",
+ "-x,y+1/2,-z+1/2",
+ "x+1/2,-y+1/2,-z",
+ ]
+ }
+ ),
+ "uses unsupported or malformed symmetry operator spelling",
+ ),
+ # Non-empty wrong-group or inconsistent display string declarations (#1025):
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_symbol": "P 1 (No. 1)"}
+ ),
+ "declares space-group number 1, which disagrees with space_group_number 19",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_symbol": "P 21 21 21 (No. 1)"}
+ ),
+ "declares space-group number 1, which disagrees with space_group_number 19",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_symbol": "P 1 (No. 19)"}
+ ),
+ "normalized symbol 'P1'\\) disagrees with input_ledger space_group_symbol 'P212121'",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_symbol": "C 21 21 21 (No. 19)"}
+ ),
+ "normalized symbol 'C212121'\\) disagrees with input_ledger space_group_symbol 'P212121'",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_symbol": "P 21 21 21"}
+ ),
+ "must match installed cctbx\\.sgtbx\\.space_group_info\\.symbol_and_number\\(\\) format",
+ ),
+ (
+ lambda p: p["crystal_symmetry"].update(
+ {"space_group_symbol": "arbitrary_nonempty_string"}
+ ),
+ "must match installed cctbx\\.sgtbx\\.space_group_info\\.symbol_and_number\\(\\) format",
+ ),
+ )):
+ with self.subTest(idx=idx_symm, symm_err=symm_err):
+ bad_symm_payload = copy.deepcopy(payload)
+ bad_symm_fn(bad_symm_payload)
+ with self.assertRaisesRegex(ValueError, symm_err):
+ self._reconcile(
+ bad_symm_payload,
+ ledger,
+ vendor_shas,
+ maps_bytes,
+ pair_bytes,
+ )
+
+ # Also verify the real committed 1SAR MTZ parser-produced ledger cell tokens normalize cleanly:
+ real_mtz_bytes = onesar_rerun.read_verified_archive_member(
+ COMMITTED_ARCHIVE,
+ onesar_rerun.MTZ_MEMBER,
+ expected_archive_sha256=onesar_rerun.ARCHIVE_SHA256,
+ expected_member_sha256=onesar_rerun.MTZ_SHA256,
+ )
+ real_proj_bytes, _ = onesar_rerun.project_observation_mtz(real_mtz_bytes)
+ real_ledger = onesar_rerun.build_input_hkl_ledger(real_proj_bytes)
+ real_cell_tokens = real_ledger["crystallographic_context"][
+ "cell_a_b_c_alpha_beta_gamma"
+ ]
+ self.assertTrue(all(type(tok) is str for tok in real_cell_tokens))
+ self.assertEqual(
+ onesar_rerun.validate_unit_cell_parameters(real_cell_tokens),
+ (64.897, 78.323, 38.792, 90.0, 90.0, 90.0),
+ )
+
+ # 1. Orthogonal cell closed-form check: (10, 20, 30, 90, 90, 90).
+ orth_cell = (10.0, 20.0, 30.0, 90.0, 90.0, 90.0)
+ self.assertAlmostEqual(
+ onesar_rerun.compute_hkl_d_spacing([1, 0, 0], orth_cell),
+ 10.0,
+ places=14,
+ )
+ self.assertAlmostEqual(
+ onesar_rerun.compute_hkl_d_spacing([0, 1, 0], orth_cell),
+ 20.0,
+ places=14,
+ )
+ self.assertAlmostEqual(
+ onesar_rerun.compute_hkl_d_spacing([0, 0, 1], orth_cell),
+ 30.0,
+ places=14,
+ )
+ self.assertAlmostEqual(
+ onesar_rerun.compute_hkl_d_spacing([1, 2, 3], orth_cell),
+ 10.0 / math.sqrt(3.0),
+ places=14,
+ )
+
+ # 2. Non-orthogonal hexagonal cell closed-form check: (10, 10, 20, 90, 90, 120).
+ # Exact formula: 1/d^2 = (4/3)*(h^2 + h*k + k^2)/a^2 + l^2/c^2.
+ hex_cell = (10.0, 10.0, 20.0, 90.0, 90.0, 120.0)
+ self.assertAlmostEqual(
+ onesar_rerun.compute_hkl_d_spacing([1, 0, 0], hex_cell),
+ 5.0 * math.sqrt(3.0),
+ places=13,
+ )
+ self.assertAlmostEqual(
+ onesar_rerun.compute_hkl_d_spacing([1, 1, 0], hex_cell),
+ 5.0,
+ places=13,
+ )
+ self.assertAlmostEqual(
+ onesar_rerun.compute_hkl_d_spacing([1, -1, 0], hex_cell),
+ 5.0 * math.sqrt(3.0),
+ places=13,
+ )
+ self.assertAlmostEqual(
+ onesar_rerun.compute_hkl_d_spacing([1, 1, 2], hex_cell),
+ 2.0 * math.sqrt(5.0),
+ places=13,
+ )
+ self.assertAlmostEqual(
+ onesar_rerun.compute_hkl_d_spacing([2, 0, 0], hex_cell),
+ 2.5 * math.sqrt(3.0),
+ places=13,
+ )
+
+ # Full reconcile_phenix_rfactor_output on a non-orthogonal hexagonal cell:
+ # fixture stage HKLs:
+ # extracted_pre_fmodel: (1,0,0)->5*sqrt(3), (0,1,0)->5*sqrt(3),
+ # (1,1,1)->1/sqrt(12/300 + 1/400)=20/sqrt(17),
+ # (2,0,0)->2.5*sqrt(3), (0,2,0)->2.5*sqrt(3)
+ # final_effective: (1,0,0), (0,1,0), (2,0,0), (0,2,0)
+ # actual_work: (1,0,0), (0,1,0) -> d = 5*sqrt(3)
+ # actual_free: (2,0,0), (0,2,0) -> d = 2.5*sqrt(3)
+ hex_mtz_bytes = _build_synthetic_mtz(
+ [
+ (1, 0, 0, 100.0, 2.0, 0.0),
+ (0, 1, 0, 200.0, 3.0, 0.0),
+ (0, 0, 1, 0.0, 1.5, 0.0),
+ (1, 1, 0, -5.0, 1.2, 0.0),
+ (1, 1, 1, 150.0, 2.5, 0.0),
+ (2, 0, 0, 80.0, 1.8, 1.0),
+ (0, 2, 0, 120.0, 2.2, 1.0),
+ (0, 0, 2, float("nan"), float("nan"), float("nan")),
+ ],
+ cell=hex_cell,
+ )
+ hex_ledger = onesar_rerun.build_input_hkl_ledger(hex_mtz_bytes)
+ hex_maps_bytes, hex_pair_bytes = _build_synthetic_map_mtz_pair(
+ payload["map_coefficients"]["2mFo-DFc"]["records"],
+ payload["map_coefficients"]["mFo-DFc"]["records"],
+ cell=hex_cell,
+ )
+ hex_payload = copy.deepcopy(payload)
+ hex_payload["crystal_symmetry"]["unit_cell"] = list(
+ onesar_rerun.ccp4_mtz_float32_unit_cell(hex_cell)
+ )
+ hex_payload["map_coefficients"]["mtz_supplements"][
+ "phx_maps_sha256"
+ ] = hashlib.sha256(hex_maps_bytes).hexdigest()
+ hex_payload["map_coefficients"]["mtz_supplements"][
+ "obs_pair_only_sha256"
+ ] = hashlib.sha256(hex_pair_bytes).hexdigest()
+ hex_d_100 = 5.0 * math.sqrt(3.0)
+ hex_d_200 = 2.5 * math.sqrt(3.0)
+ hex_d_111 = 20.0 / math.sqrt(17.0)
+ hex_payload["resolution_ranges"] = {
+ "extracted_pre_fmodel": {"d_max": hex_d_100, "d_min": hex_d_200},
+ "final_effective": {"d_max": hex_d_100, "d_min": hex_d_200},
+ "actual_work": {"d_max": hex_d_100, "d_min": hex_d_100},
+ "actual_free": {"d_max": hex_d_200, "d_min": hex_d_200},
+ }
+ self.assertAlmostEqual(
+ onesar_rerun.compute_hkl_d_spacing([1, 1, 1], hex_cell),
+ hex_d_111,
+ places=13,
+ )
+ hex_res = self._reconcile(
+ hex_payload,
+ hex_ledger,
+ vendor_shas,
+ hex_maps_bytes,
+ hex_pair_bytes,
+ )
+ self.assertEqual(hex_res["reconciliation_status"], "verified")
+ self.assertAlmostEqual(
+ hex_res["resolution_ranges"]["actual_work"]["recomputed_d_max"],
+ hex_d_100,
+ places=12,
+ )
+
+ # Orthogonal approximation on the hexagonal cell (10.0 instead of 5*sqrt(3)) must fail!
+ hex_orth_approx = copy.deepcopy(hex_payload)
+ hex_orth_approx["resolution_ranges"]["actual_work"] = {
+ "d_max": 10.0,
+ "d_min": 10.0,
+ }
+ with self.assertRaisesRegex(
+ ValueError,
+ "resolution_ranges\\['actual_work'\\]\\.d_max .* disagrees with recomputed",
+ ):
+ self._reconcile(
+ hex_orth_approx,
+ hex_ledger,
+ vendor_shas,
+ hex_maps_bytes,
+ hex_pair_bytes,
+ )
+
+ # 3. General triclinic cell verified against independent 3D Cartesian cross-product basis:
+ tri_cell = (10.0, 12.0, 15.0, 80.0, 85.0, 70.0)
+ ta, tb, tc, talpha, tbeta, tgamma = tri_cell
+ ca_r = math.cos(math.radians(talpha))
+ cb_r = math.cos(math.radians(tbeta))
+ cg_r = math.cos(math.radians(tgamma))
+ sg_r = math.sin(math.radians(tgamma))
+ vec_a = (ta, 0.0, 0.0)
+ vec_b = (tb * cg_r, tb * sg_r, 0.0)
+ cx = tc * cb_r
+ cy = tc * (ca_r - cb_r * cg_r) / sg_r
+ cz = math.sqrt(tc * tc - cx * cx - cy * cy)
+ vec_c = (cx, cy, cz)
+
+ def _cross(
+ u: tuple[float, float, float], v: tuple[float, float, float]
+ ) -> tuple[float, float, float]:
+ return (
+ u[1] * v[2] - u[2] * v[1],
+ u[2] * v[0] - u[0] * v[2],
+ u[0] * v[1] - u[1] * v[0],
+ )
+
+ def _dot(
+ u: tuple[float, float, float], v: tuple[float, float, float]
+ ) -> float:
+ return u[0] * v[0] + u[1] * v[1] + u[2] * v[2]
+
+ vol = _dot(vec_a, _cross(vec_b, vec_c))
+ b_cross_c = _cross(vec_b, vec_c)
+ c_cross_a = _cross(vec_c, vec_a)
+ a_cross_b = _cross(vec_a, vec_b)
+ astar = (b_cross_c[0] / vol, b_cross_c[1] / vol, b_cross_c[2] / vol)
+ bstar = (c_cross_a[0] / vol, c_cross_a[1] / vol, c_cross_a[2] / vol)
+ cstar = (a_cross_b[0] / vol, a_cross_b[1] / vol, a_cross_b[2] / vol)
+
+ tri_hkls = [
+ [1, 0, 0],
+ [0, 1, 0],
+ [0, 0, 1],
+ [1, 2, -1],
+ [-2, 1, 3],
+ [3, -2, -4],
+ ]
+ indep_ds: list[float] = []
+ for h, k, l in tri_hkls:
+ sx = h * astar[0] + k * bstar[0] + l * cstar[0]
+ sy = h * astar[1] + k * bstar[1] + l * cstar[1]
+ sz = h * astar[2] + k * bstar[2] + l * cstar[2]
+ d_indep = 1.0 / math.sqrt(sx * sx + sy * sy + sz * sz)
+ indep_ds.append(d_indep)
+ self.assertAlmostEqual(
+ onesar_rerun.compute_hkl_d_spacing([h, k, l], tri_cell),
+ d_indep,
+ places=13,
+ )
+ tri_stage_range = onesar_rerun.compute_stage_resolution_range(
+ tri_hkls, tri_cell
+ )
+ self.assertAlmostEqual(
+ tri_stage_range["d_max"], max(indep_ds), places=13
+ )
+ self.assertAlmostEqual(
+ tri_stage_range["d_min"], min(indep_ds), places=13
+ )
+
+ # 4. Missing, malformed, swapped, nonfinite, boolean, cross-stage copied, or fabricated ranges fail:
+ missing_rr = copy.deepcopy(payload)
+ del missing_rr["resolution_ranges"]
+ with self.assertRaisesRegex(
+ ValueError, "phenix_payload\\.resolution_ranges must be a mapping"
+ ):
+ self._reconcile(
+ missing_rr, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ missing_stage = copy.deepcopy(payload)
+ del missing_stage["resolution_ranges"]["actual_free"]
+ with self.assertRaisesRegex(
+ ValueError, "phenix_payload\\.resolution_ranges must be a mapping"
+ ):
+ self._reconcile(
+ missing_stage, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ extra_stage = copy.deepcopy(payload)
+ extra_stage["resolution_ranges"]["raw_mtz"] = {
+ "d_max": 78.323,
+ "d_min": 64.897,
+ }
+ with self.assertRaisesRegex(
+ ValueError, "phenix_payload\\.resolution_ranges must be a mapping"
+ ):
+ self._reconcile(
+ extra_stage, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ missing_dmin = copy.deepcopy(payload)
+ del missing_dmin["resolution_ranges"]["actual_work"]["d_min"]
+ with self.assertRaisesRegex(
+ ValueError,
+ "resolution_ranges\\['actual_work'\\] must be a mapping with exact keys",
+ ):
+ self._reconcile(
+ missing_dmin, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ swapped_bounds = copy.deepcopy(payload)
+ swapped_bounds["resolution_ranges"]["actual_work"] = {
+ "d_max": 64.897,
+ "d_min": 78.323,
+ }
+ with self.assertRaisesRegex(
+ ValueError, "has inverted bounds: d_max=64.897 < d_min=78.323"
+ ):
+ self._reconcile(
+ swapped_bounds, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ for bad_val, err_pat in (
+ (float("nan"), "must be a finite number"),
+ (float("inf"), "must be a finite number"),
+ (0.0, "must be positive"),
+ (-5.0, "must be positive"),
+ (True, "must be a finite number"),
+ ):
+ bad_d = copy.deepcopy(payload)
+ bad_d["resolution_ranges"]["actual_work"]["d_min"] = bad_val
+ with self.assertRaisesRegex(ValueError, err_pat):
+ self._reconcile(
+ bad_d, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # Copying final_effective onto extracted_pre_fmodel, actual_work, or actual_free fails:
+ for stage_target, field_pat in (
+ ("extracted_pre_fmodel", "d_min"),
+ ("actual_work", "d_min"),
+ ("actual_free", "d_max"),
+ ):
+ copied_stage = copy.deepcopy(payload)
+ copied_stage["resolution_ranges"][stage_target] = copy.deepcopy(
+ payload["resolution_ranges"]["final_effective"]
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ f"resolution_ranges\\['{stage_target}'\\]\\.{field_pat} .* disagrees with recomputed",
+ ):
+ self._reconcile(
+ copied_stage, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # Fabricated 1.8 A range on the low-resolution fixture stage HKL set fails:
+ fabricated_18 = copy.deepcopy(payload)
+ fabricated_18["resolution_ranges"]["final_effective"]["d_min"] = 1.8
+ with self.assertRaisesRegex(
+ ValueError,
+ "resolution_ranges\\['final_effective'\\]\\.d_min 1\\.800000000000000 disagrees with recomputed",
+ ):
+ self._reconcile(
+ fabricated_18, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # Numeric scalar/measurement checks remain strict (do not accept decimal strings):
+ str_d = copy.deepcopy(payload)
+ str_d["resolution_ranges"]["actual_work"]["d_min"] = "64.897"
+ with self.assertRaisesRegex(ValueError, "must be a finite number"):
+ self._reconcile(
+ str_d, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+ str_rw = copy.deepcopy(payload)
+ str_rw["scales_and_rfactors"]["r_work"] = "0.06666666666666667"
+ with self.assertRaisesRegex(ValueError, "must be a finite number"):
+ self._reconcile(
+ str_rw, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # 5. Origin (0, 0, 0) HKL and invalid/singular/malformed-token unit cells raise ValueError:
+ with self.assertRaisesRegex(
+ ValueError, "Miller index \\(0, 0, 0\\) has undefined d-spacing"
+ ):
+ onesar_rerun.compute_hkl_d_spacing([0, 0, 0], orth_cell)
+ with self.assertRaisesRegex(
+ ValueError, "Miller index \\(0, 0, 0\\) has undefined d-spacing"
+ ):
+ onesar_rerun.compute_stage_resolution_range([[0, 0, 0]], orth_cell)
+
+ for bad_cell, cell_err in (
+ ([10.0, 20.0, 30.0], "must be a 6-element sequence"),
+ ((0.0, 20.0, 30.0, 90.0, 90.0, 90.0), "must be positive"),
+ ((-10.0, 20.0, 30.0, 90.0, 90.0, 90.0), "must be positive"),
+ ((True, 20.0, 30.0, 90.0, 90.0, 90.0), "must be a finite number"),
+ ((10.0, 20.0, 30.0, 0.0, 90.0, 90.0), "must be positive"),
+ (
+ (10.0, 20.0, 30.0, 180.0, 90.0, 90.0),
+ "must be in \\(0, 180\\) degrees",
+ ),
+ (
+ (10.0, 20.0, 30.0, 90.0, 120.0, 150.0),
+ "violate crystallographic cone inequalities",
+ ),
+ (
+ (10.0, 20.0, 30.0, 40.0, 50.0, 100.0),
+ "violate crystallographic cone inequalities",
+ ),
+ (
+ ("", "78.323", "38.792", "9E+1", "9E+1", "9E+1"),
+ "must be a valid decimal unit-cell token",
+ ),
+ (
+ (" 64.897", "78.323", "38.792", "9E+1", "9E+1", "9E+1"),
+ "must be a valid decimal unit-cell token",
+ ),
+ (
+ ("64.897 ", "78.323", "38.792", "9E+1", "9E+1", "9E+1"),
+ "must be a valid decimal unit-cell token",
+ ),
+ (
+ ("64_897", "78.323", "38.792", "9E+1", "9E+1", "9E+1"),
+ "must be a valid decimal unit-cell token",
+ ),
+ (
+ ("64.897a", "78.323", "38.792", "9E+1", "9E+1", "9E+1"),
+ "must be a valid decimal unit-cell token",
+ ),
+ (
+ ("NaN", "78.323", "38.792", "9E+1", "9E+1", "9E+1"),
+ "must be a finite number",
+ ),
+ (
+ ("inf", "78.323", "38.792", "9E+1", "9E+1", "9E+1"),
+ "must be a finite number",
+ ),
+ (
+ ("-Infinity", "78.323", "38.792", "9E+1", "9E+1", "9E+1"),
+ "must be a finite number",
+ ),
+ (
+ ("1e1000", "78.323", "38.792", "9E+1", "9E+1", "9E+1"),
+ "must be a finite number",
+ ),
+ (
+ ("0", "78.323", "38.792", "9E+1", "9E+1", "9E+1"),
+ "must be positive",
+ ),
+ (
+ ("-64.897", "78.323", "38.792", "9E+1", "9E+1", "9E+1"),
+ "must be positive",
+ ),
+ (
+ ("64.897", "78.323", "38.792", "1.8E+2", "9E+1", "9E+1"),
+ "must be in \\(0, 180\\) degrees",
+ ),
+ ):
+ with self.assertRaisesRegex(ValueError, cell_err):
+ onesar_rerun.validate_unit_cell_parameters(bad_cell)
+ bad_ledger = copy.deepcopy(ledger)
+ bad_ledger["crystallographic_context"][
+ "cell_a_b_c_alpha_beta_gamma"
+ ] = list(bad_cell)
+ with self.assertRaisesRegex(ValueError, cell_err):
+ self._reconcile(
+ payload, bad_ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ def test_map_coefficients_and_mtz_supplements_reconciliation_1016(self) -> None:
+ ledger, payload, vendor_shas, maps_bytes, pair_bytes = (
+ _make_reconciliation_fixture()
+ )
+
+ # Mutated JSON map amplitude or phase fails.
+ bad_json_amp = copy.deepcopy(payload)
+ bad_json_amp["map_coefficients"]["2mFo-DFc"]["records"][0][
+ "amplitude"
+ ] = 11.0
+ with self.assertRaisesRegex(
+ ValueError, "amplitude 11.0 != hypot\\(real, imag\\)"
+ ):
+ self._reconcile(
+ bad_json_amp, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ bad_json_phase = copy.deepcopy(payload)
+ bad_json_phase["map_coefficients"]["2mFo-DFc"]["records"][2][
+ "phase_deg"
+ ] = 45.0
+ with self.assertRaisesRegex(
+ ValueError, "phase_deg 45.0 disagrees with atan2"
+ ):
+ self._reconcile(
+ bad_json_phase, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # Mutated centric_count fails.
+ bad_centric = copy.deepcopy(payload)
+ bad_centric["map_coefficients"]["2mFo-DFc"]["centric_count"] = 3
+ bad_centric["map_coefficients"]["2mFo-DFc"]["acentric_count"] = 1
+ with self.assertRaisesRegex(
+ ValueError, "centric/acentric counts .* !="
+ ):
+ self._reconcile(
+ bad_centric, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # Wrong MTZ SHA-256 fails.
+ bad_sha = copy.deepcopy(payload)
+ bad_sha["map_coefficients"]["mtz_supplements"]["phx_maps_sha256"] = (
+ "0" * 64
+ )
+ with self.assertRaisesRegex(
+ ValueError, "phx_maps.mtz SHA-256 .* disagrees"
+ ):
+ self._reconcile(
+ bad_sha, ledger, vendor_shas, maps_bytes, pair_bytes
+ )
+
+ # Wrong column type in phx_maps.mtz (e.g. Q instead of P) fails.
+ bad_type_maps = _build_synthetic_mtz(
+ [(1, 0, 0, 10.0, 0.0, 2.0, 0.0)],
+ columns=[
+ ("H", "H", "1"),
+ ("K", "H", "1"),
+ ("L", "H", "1"),
+ ("2FOFCWT", "F", "1"),
+ ("PH2FOFCWT", "Q", "1"),
+ ("FOFCWT", "F", "1"),
+ ("PHFOFCWT", "P", "1"),
+ ],
+ )
+ bad_type_payload = copy.deepcopy(payload)
+ bad_type_payload["map_coefficients"]["mtz_supplements"][
+ "phx_maps_sha256"
+ ] = hashlib.sha256(bad_type_maps).hexdigest()
+ with self.assertRaisesRegex(
+ ValueError, "Map MTZ column types .* do not match expected"
+ ):
+ self._reconcile(
+ bad_type_payload,
+ ledger,
+ vendor_shas,
+ bad_type_maps,
+ pair_bytes,
+ )
+
+ # Mutated float32 amplitude in phx_maps.mtz fails.
+ mut_recs_2fofc = copy.deepcopy(
+ payload["map_coefficients"]["2mFo-DFc"]["records"]
+ )
+ mut_recs_2fofc[0]["amplitude"] = 99.0
+ bad_amp_maps, _ = _build_synthetic_map_mtz_pair(
+ mut_recs_2fofc, payload["map_coefficients"]["mFo-DFc"]["records"]
+ )
+ bad_amp_payload = copy.deepcopy(payload)
+ bad_amp_payload["map_coefficients"]["mtz_supplements"][
+ "phx_maps_sha256"
+ ] = hashlib.sha256(bad_amp_maps).hexdigest()
+ # Keep PHX_OBS_PAIR_ONLY matching bad_amp_maps so it reaches the JSON amplitude comparison:
+ _, bad_amp_pair = _build_synthetic_map_mtz_pair(
+ mut_recs_2fofc, payload["map_coefficients"]["mFo-DFc"]["records"]
+ )
+ bad_amp_payload["map_coefficients"]["mtz_supplements"][
+ "obs_pair_only_sha256"
+ ] = hashlib.sha256(bad_amp_pair).hexdigest()
+ with self.assertRaisesRegex(
+ ValueError, "phx_maps.mtz 2FOFCWT 99.0 disagrees with JSON amplitude 10.0"
+ ):
+ self._reconcile(
+ bad_amp_payload,
+ ledger,
+ vendor_shas,
+ bad_amp_maps,
+ bad_amp_pair,
+ )
+
+ # Mismatched 2FOFCWT word in PHX_OBS_PAIR_ONLY.mtz vs phx_maps.mtz fails.
+ mismatch_pair_payload = copy.deepcopy(payload)
+ mismatch_pair_payload["map_coefficients"]["mtz_supplements"][
+ "obs_pair_only_sha256"
+ ] = hashlib.sha256(bad_amp_pair).hexdigest()
+ with self.assertRaisesRegex(
+ ValueError,
+ "PHX_OBS_PAIR_ONLY.mtz 2FOFCWT/PH2FOFCWT words differ from phx_maps.mtz",
+ ):
+ self._reconcile(
+ mismatch_pair_payload,
+ ledger,
+ vendor_shas,
+ maps_bytes,
+ bad_amp_pair,
+ )
+
+ def test_helper_pure_functions_and_python39_static_contract(self) -> None:
+ # Pure scatterer snapshot comparison catches mutations.
+ base_rec = [
+ {
+ "label": "N",
+ "scattering_type": "N",
+ "site_frac": [0.1, 0.2, 0.3],
+ "site_cart": [6.4, 15.6, 11.6],
+ "occupancy": 1.0,
+ "u_iso": 0.25,
+ "u_star": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
+ "use_u_iso": True,
+ "use_u_aniso": False,
+ }
+ ]
+ snap_before = onesar_phenix_measure.snapshot_scatterer_records(base_rec)
+ snap_same = onesar_phenix_measure.snapshot_scatterer_records(base_rec)
+ inv = onesar_phenix_measure.compare_scatterer_snapshots(
+ snap_before, snap_same, snap_same
+ )
+ self.assertEqual(inv["max_abs_site_cart_diff"], 0.0)
+
+ mut_rec = copy.deepcopy(base_rec)
+ mut_rec[0]["site_cart"][0] += 0.001
+ snap_mut = onesar_phenix_measure.snapshot_scatterer_records(mut_rec)
+ with self.assertRaisesRegex(ValueError, "Scatterer snapshot mutated"):
+ onesar_phenix_measure.compare_scatterer_snapshots(
+ snap_before, snap_mut, snap_same
+ )
+
+ # build_raw_scatterer_snapshot retains empty and non-finite records without
+ # validating finiteness or computing a canonical digest (#1014, Task 128):
+ raw_empty_snap = onesar_phenix_measure.build_raw_scatterer_snapshot([])
+ self.assertEqual(
+ raw_empty_snap,
+ {"n_scatterers": 0, "sha256": None, "records": []},
+ )
+ with self.subTest(
+ case="strict_snapshot_rejects_empty_scatterer_records"
+ ):
+ with self.assertRaisesRegex(
+ ValueError, "Scatterer snapshot cannot be empty"
+ ):
+ onesar_phenix_measure.snapshot_scatterer_records([])
+
+ nf_occ_bits = struct.unpack(
+ ">d", struct.pack(">Q", 0x7FF8000000000123)
+ )[0]
+ nf_rec = copy.deepcopy(base_rec)
+ nf_rec[0]["occupancy"] = nf_occ_bits
+ raw_nf_snap = onesar_phenix_measure.build_raw_scatterer_snapshot(nf_rec)
+ self.assertEqual(raw_nf_snap["n_scatterers"], 1)
+ self.assertIsNone(raw_nf_snap["sha256"])
+ encoded_raw_nf_snap = (
+ onesar_phenix_measure.encode_raw_evidence_nonfinite(
+ raw_nf_snap,
+ field_path="xray_structure_snapshots_raw.after_model",
+ )
+ )
+ self.assertEqual(
+ encoded_raw_nf_snap["records"][0]["occupancy"][
+ "ieee754_binary64_hex"
+ ],
+ "0x7ff8000000000123",
+ )
+ with self.subTest(
+ case="strict_snapshot_rejects_nonfinite_scatterer_record"
+ ):
+ with self.assertRaisesRegex(
+ ValueError, "Invalid scatterer record at index 0"
+ ):
+ onesar_phenix_measure.snapshot_scatterer_records(nf_rec)
+
+ # Symmetry operator normalization policy (#1025, Task 128):
+ self.assertEqual(
+ onesar_rerun.normalize_symmetry_operator_xyz(
+ " -x + 1/2, -y, z + 1/2 "
+ ),
+ "-X+1/2,-Y,Z+1/2",
+ )
+ for bad_op_case, bad_op_str, bad_op_pat in (
+ (
+ "symm_prefixed",
+ "SYMM -x + 1/2, -y, z + 1/2 ",
+ "unsupported or malformed",
+ ),
+ (
+ "translation_first",
+ "1/2-X, -Y, 1/2+Z",
+ "unsupported or malformed",
+ ),
+ (
+ "leading_plus_translation_first",
+ "+1/2+X, +1/2-Y, -Z",
+ "unsupported or malformed",
+ ),
+ (
+ "wrong_component_count",
+ "X,Y",
+ "must have 3 comma-separated XYZ components",
+ ),
+ (
+ "decimal_translation",
+ "X+0.5,Y,Z",
+ "unsupported or malformed",
+ ),
+ ):
+ with self.subTest(case=f"normalize_symmetry_rejects_{bad_op_case}"):
+ with self.assertRaisesRegex(ValueError, bad_op_pat):
+ onesar_rerun.normalize_symmetry_operator_xyz(bad_op_str)
+
+ # Strict-JSON IEEE-754 nonfinite tagger preserves finite values and tags NaN/+Inf/-Inf:
+ nf_locs: list[dict[str, Any]] = []
+ encoded_nf = onesar_phenix_measure.encode_raw_evidence_nonfinite(
+ {
+ "finite": 12.5,
+ "flag": True,
+ "count": 3,
+ "items": [float("nan"), float("inf"), float("-inf")],
+ },
+ nonfinite_locations=nf_locs,
+ )
+ self.assertEqual(encoded_nf["finite"], 12.5)
+ self.assertIs(encoded_nf["flag"], True)
+ self.assertEqual(encoded_nf["count"], 3)
+ self.assertEqual(len(nf_locs), 3)
+ self.assertEqual(
+ [item["kind"] for item in nf_locs], ["nan", "+inf", "-inf"]
+ )
+ self.assertEqual(
+ [item["field_path"] for item in nf_locs],
+ ["items[0]", "items[1]", "items[2]"],
+ )
+ self.assertEqual(
+ nf_locs[1]["ieee754_binary64_hex"], "0x7ff0000000000000"
+ )
+ self.assertEqual(
+ nf_locs[2]["ieee754_binary64_hex"], "0xfff0000000000000"
+ )
+ # Re-encoding an already tagged structure is idempotent:
+ nf_locs_second: list[dict[str, Any]] = []
+ encoded_nf_second = (
+ onesar_phenix_measure.encode_raw_evidence_nonfinite(
+ encoded_nf, nonfinite_locations=nf_locs_second
+ )
+ )
+ self.assertEqual(encoded_nf_second, encoded_nf)
+ self.assertEqual(nf_locs_second, nf_locs)
+
+ # Static AST verification of scripts/onesar_phenix_measure.py:
+ # - No top-level vendor imports
+ # - No Python 3.10+/3.11+ zip(strict=...) or hashlib.file_digest
+ # - No call to second-block get_r_rfree_sigma
+ helper_path = _SCRIPTS_DIR / "onesar_phenix_measure.py"
+ tree = ast.parse(helper_path.read_text(encoding="utf-8"))
+ forbidden_top_imports = {"cctbx", "iotbx", "mmtbx", "scitbx", "libtbx"}
+ for node in tree.body:
+ if isinstance(node, ast.Import):
+ for alias in node.names:
+ root_mod = alias.name.split(".")[0]
+ self.assertNotIn(root_mod, forbidden_top_imports)
+ elif isinstance(node, ast.ImportFrom) and node.module:
+ root_mod = node.module.split(".")[0]
+ self.assertNotIn(root_mod, forbidden_top_imports)
+
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Call):
+ if isinstance(node.func, ast.Attribute):
+ self.assertNotEqual(node.func.attr, "file_digest")
+ self.assertNotEqual(node.func.attr, "get_r_rfree_sigma")
+ if (
+ isinstance(node.func.value, ast.Name)
+ and node.func.value.id == "reflection_file"
+ ):
+ self.assertNotEqual(node.func.attr, "crystal_symmetry")
+ if (
+ isinstance(node.func.value, ast.Name)
+ and node.func.value.id == "extract_xtal_data"
+ and node.func.attr == "run"
+ ):
+ kw_names = {kw.arg for kw in node.keywords}
+ self.assertNotIn("log", kw_names)
+ if (
+ isinstance(node.func.value, ast.Attribute)
+ and node.func.value.attr == "f_model"
+ and node.func.attr == "manager"
+ ):
+ kw_names = {kw.arg for kw in node.keywords}
+ self.assertNotIn("log", kw_names)
+ if node.func.attr == "map_coefficients":
+ kw_names = {kw.arg for kw in node.keywords}
+ self.assertIn("sharp", kw_names)
+ self.assertNotIn("resharp_after_isotropize", kw_names)
+ elif isinstance(node.func, ast.Name) and node.func.id == "zip":
+ kw_names = {kw.arg for kw in node.keywords}
+ self.assertNotIn("strict", kw_names)
+
+ def test_operation_rejects_protstruct_env_overrides(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ fake_repo = Path(tmp).resolve(strict=True)
+ with mock.patch.dict(
+ os.environ, {"PROTSTRUCT_PHENIX_BIN": "/custom/phenix_bin"}
+ ):
+ with self.assertRaisesRegex(
+ ValueError, "no PROTSTRUCT_\\* environment overrides"
+ ):
+ onesar_rerun.run_phenix_rfactor_operation(
+ fake_repo / "out", subject_id="R4", repo_root=fake_repo
+ )
+
+ def test_stage_subject_inputs_and_unmeasured_build_rejection(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ fake_repo = Path(tmp).resolve(strict=True)
+ staged = onesar_rerun.stage_subject_inputs(
+ fake_repo / "runs" / "r4_stage",
+ subject_id="R4",
+ repo_root=fake_repo,
+ archive_path=COMMITTED_ARCHIVE,
+ )
+ self.assertTrue(staged["original_pdb_path"].is_file())
+ self.assertTrue(staged["original_mtz_path"].is_file())
+ self.assertTrue(staged["projected_mtz_path"].is_file())
+ self.assertTrue(staged["input_ledger_path"].is_file())
+ self.assertTrue(staged["admission_json_path"].is_file())
+ self.assertEqual(
+ staged["admission_record"]["subject"]["sha256"],
+ onesar_rerun.ALLOWED_SUBJECTS["R4"]["sha256"],
+ )
+ self.assertEqual(
+ staged["input_ledger"]["counts"]["eligible_total"], 7228
+ )
+ persisted_ledger = json.loads(
+ staged["input_ledger_path"].read_text(encoding="utf-8")
+ )
+ self.assertEqual(
+ persisted_ledger["counts"], staged["input_ledger"]["counts"]
+ )
+ self.assertEqual(
+ len(persisted_ledger["eligible_by_hkl"]), 7228
+ )
+ self.assertEqual(len(persisted_ledger["ineligible_rows"]), 20)
+ self.assertEqual(len(persisted_ledger["raw_rows"]), 7248)
+
+ # Rejects unmeasured PHENIX build dictionary before invoking subprocess (#1014).
+ unmeasured_build = {
+ "reported_version_source": "command_output",
+ "reported_version": "2.0-5936",
+ "parsed_build": "2.0-5936",
+ "version_probe": {"probe_mode": "command_output"},
+ }
+ unmeasured_runner_calls: list[list[str]] = []
+
+ def _unexpected_runner(
+ argv: list[str], log: Path, *, cwd: Path, timeout: float
+ ) -> subprocess.CompletedProcess[str]:
+ unmeasured_runner_calls.append(list(argv))
+ raise AssertionError("Scientific worker must not be launched")
+
+ unmeasured_run_dir = fake_repo / "runs" / "r4_unmeasured"
+ with mock.patch.object(
+ toolchain, "phenix_build_evidence", return_value=unmeasured_build
+ ):
+ with self.assertRaisesRegex(
+ ValueError, "failed #957 admission gate"
+ ):
+ onesar_rerun.run_phenix_rfactor_operation(
+ unmeasured_run_dir,
+ subject_id="R4",
+ repo_root=fake_repo,
+ archive_path=COMMITTED_ARCHIVE,
+ require_no_protstruct_overrides=False,
+ runner=_unexpected_runner,
+ )
+ self.assertEqual(unmeasured_runner_calls, [])
+ self.assertFalse((unmeasured_run_dir / "phenix" / "cache").exists())
+ preflight_unmeasured_path = (
+ unmeasured_run_dir
+ / "phenix"
+ / "phenix_preflight_build_evidence.json"
+ )
+ failure_unmeasured_path = (
+ unmeasured_run_dir
+ / "phenix"
+ / "phenix_build_admission_failure.json"
+ )
+ self.assertTrue(preflight_unmeasured_path.is_file())
+ self.assertTrue(failure_unmeasured_path.is_file())
+ self.assertEqual(
+ json.loads(preflight_unmeasured_path.read_text(encoding="utf-8")),
+ unmeasured_build,
+ )
+ failure_unmeasured_doc = json.loads(
+ failure_unmeasured_path.read_text(encoding="utf-8")
+ )
+ self.assertEqual(failure_unmeasured_doc["status"], "rejected")
+ self.assertFalse(failure_unmeasured_doc["worker_launched"])
+ self.assertEqual(
+ failure_unmeasured_doc["measured_build"], unmeasured_build
+ )
+ self.assertEqual(
+ failure_unmeasured_doc["failure"]["failed_stage"],
+ "validate_measured_phenix_build",
+ )
+ self.assertEqual(
+ failure_unmeasured_doc["preflight_build_evidence_sha256"],
+ hashlib.sha256(preflight_unmeasured_path.read_bytes()).hexdigest(),
+ )
+
+ # Rejects failed version_probe and retains full raw probe evidence on disk (#1014).
+ failed_probe_build = {
+ "reported_version_source": "unknown",
+ "reported_version": "unknown",
+ "parsed_build": None,
+ "version_probe": {
+ "error": "phenix_env.sh exited with returncode 1",
+ "probe_mode": "vendor_setup_environment",
+ "setup_returncode": 1,
+ "setup_stdout": "",
+ "setup_stderr": "phenix_env.sh: missing dispatcher",
+ },
+ }
+ failed_probe_run_dir = fake_repo / "runs" / "r4_failed_probe"
+ with mock.patch.object(
+ toolchain,
+ "phenix_build_evidence",
+ return_value=failed_probe_build,
+ ):
+ with self.assertRaisesRegex(
+ ValueError, "failed #957 admission gate"
+ ):
+ onesar_rerun.run_phenix_rfactor_operation(
+ failed_probe_run_dir,
+ subject_id="R4",
+ repo_root=fake_repo,
+ archive_path=COMMITTED_ARCHIVE,
+ require_no_protstruct_overrides=False,
+ runner=_unexpected_runner,
+ )
+ self.assertEqual(unmeasured_runner_calls, [])
+ self.assertFalse(
+ (failed_probe_run_dir / "phenix" / "cache").exists()
+ )
+ preflight_failed_path = (
+ failed_probe_run_dir
+ / "phenix"
+ / "phenix_preflight_build_evidence.json"
+ )
+ admission_failed_path = (
+ failed_probe_run_dir
+ / "phenix"
+ / "phenix_build_admission_failure.json"
+ )
+ self.assertTrue(preflight_failed_path.is_file())
+ self.assertTrue(admission_failed_path.is_file())
+ self.assertEqual(
+ json.loads(preflight_failed_path.read_text(encoding="utf-8")),
+ failed_probe_build,
+ )
+ admission_failed_doc = json.loads(
+ admission_failed_path.read_text(encoding="utf-8")
+ )
+ self.assertEqual(admission_failed_doc["status"], "rejected")
+ self.assertFalse(admission_failed_doc["worker_launched"])
+ self.assertEqual(
+ admission_failed_doc["measured_build"], failed_probe_build
+ )
+ self.assertEqual(
+ admission_failed_doc["failure"]["failed_stage"],
+ "validate_measured_phenix_build",
+ )
+
+ def test_orchestration_with_real_cache_success_and_failure_retention(
+ self,
+ ) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ tmp_root = Path(tmp).resolve(strict=True)
+ fake_repo = tmp_root / "repo"
+ fake_repo.mkdir()
+
+ # Build a synthetic PHENIX installation tree for file_identity and vendor source hashing.
+ fake_prefix = tmp_root / "phenix-2.0-5936"
+ fake_bin = fake_prefix / "phenix_bin"
+ fake_site = fake_prefix / "lib" / "python3.9" / "site-packages"
+ fake_bin.mkdir(parents=True)
+ (fake_prefix / "phenix_env.sh").write_text(
+ "#!/bin/sh\nexport PHENIX_VERSION=2.0\nexport PHENIX_RELEASE_TAG=5936\n",
+ encoding="utf-8",
+ )
+ (fake_bin / "phenix.version").write_text(
+ "#!/bin/sh\n", encoding="utf-8"
+ )
+ (fake_bin / "phenix.python").write_text(
+ "#!/bin/sh\n", encoding="utf-8"
+ )
+
+ vendor_modules_payload = {}
+ for key, rel in onesar_rerun.PHENIX_VENDOR_SOURCE_RELPATHS.items():
+ mod_file = fake_site / rel
+ mod_file.parent.mkdir(parents=True, exist_ok=True)
+ content = f"# synthetic {key}\n".encode("utf-8")
+ mod_file.write_bytes(content)
+ resolved_mod = mod_file.resolve(strict=True)
+ vendor_modules_payload[key] = {
+ "module_name": key,
+ "configured_path": str(resolved_mod),
+ "resolved_path": str(resolved_mod),
+ "sha256": hashlib.sha256(content).hexdigest(),
+ }
+
+ banner = "PHENIX: 2.0-5936\n Version: 2.0\n Release tag: 5936"
+ setup_resolved = (fake_prefix / "phenix_env.sh").resolve(strict=True)
+ disp_resolved = (fake_bin / "phenix.version").resolve(strict=True)
+ resolved_prefix_str = str(fake_prefix.resolve(strict=True))
+ fake_measured_build = {
+ "reported_version_source": "vendor_setup_environment",
+ "reported_version": banner,
+ "parsed_build": "2.0-5936",
+ "version_probe": {
+ "error": None,
+ "probe_mode": "vendor_setup_environment",
+ "configured_prefix": str(fake_prefix),
+ "resolved_prefix": resolved_prefix_str,
+ "setup_configured_path": str(fake_prefix / "phenix_env.sh"),
+ "setup_resolved_path": str(setup_resolved),
+ "setup_sha256": hashlib.sha256(
+ setup_resolved.read_bytes()
+ ).hexdigest(),
+ "setup_argv": toolchain.phenix_setup_argv(setup_resolved),
+ "setup_returncode": 0,
+ "setup_stdout": "",
+ "setup_stderr": "",
+ "setup_version_fields": {
+ "PHENIX_VERSION": "2.0",
+ "PHENIX_RELEASE_TAG": "5936",
+ },
+ "setup_parsed_build": "2.0-5936",
+ "configured_path": str(fake_bin / "phenix.version"),
+ "resolved_path": str(disp_resolved),
+ "sha256": hashlib.sha256(
+ disp_resolved.read_bytes()
+ ).hexdigest(),
+ "argv": [str(disp_resolved)],
+ "returncode": 0,
+ "stdout": banner + "\n",
+ "stderr": "",
+ "parsed_build": "2.0-5936",
+ "ignored_inherited_version_variables": [],
+ "ignored_inherited_shell_hooks": [],
+ },
+ }
+ self.assertTrue(
+ refinement_cache._build_is_measured(fake_measured_build)
+ )
+
+ # Internally consistent wrong version (2.0-5935) must be rejected (#1013, #1014)
+ # and retain preflight build evidence + admission failure without launching worker.
+ wrong_ver_banner = "PHENIX: 2.0-5935\n Version: 2.0\n Release tag: 5935"
+ wrong_ver_build = copy.deepcopy(fake_measured_build)
+ wrong_ver_build["reported_version"] = wrong_ver_banner
+ wrong_ver_build["parsed_build"] = "2.0-5935"
+ wrong_ver_build["version_probe"]["setup_version_fields"] = {
+ "PHENIX_VERSION": "2.0",
+ "PHENIX_RELEASE_TAG": "5935",
+ }
+ wrong_ver_build["version_probe"]["setup_parsed_build"] = "2.0-5935"
+ wrong_ver_build["version_probe"]["stdout"] = wrong_ver_banner + "\n"
+ wrong_ver_build["version_probe"]["parsed_build"] = "2.0-5935"
+ self.assertTrue(refinement_cache._build_is_measured(wrong_ver_build))
+ wrong_ver_runner_calls: list[list[str]] = []
+
+ def _wrong_ver_spy_runner(
+ argv: list[str], log: Path, *, cwd: Path, timeout: float
+ ) -> subprocess.CompletedProcess[str]:
+ wrong_ver_runner_calls.append(list(argv))
+ raise AssertionError("Scientific worker must not be launched")
+
+ wrong_ver_run_dir = fake_repo / "runs" / "r4_wrong_ver"
+ with mock.patch.object(
+ toolchain, "phenix_build_evidence", return_value=wrong_ver_build
+ ):
+ with self.assertRaisesRegex(
+ ValueError, "PHENIX build version mismatch: expected '2.0-5936'"
+ ):
+ onesar_rerun.run_phenix_rfactor_operation(
+ wrong_ver_run_dir,
+ subject_id="R4",
+ repo_root=fake_repo,
+ archive_path=COMMITTED_ARCHIVE,
+ require_no_protstruct_overrides=False,
+ runner=_wrong_ver_spy_runner,
+ )
+ self.assertEqual(wrong_ver_runner_calls, [])
+ self.assertFalse((wrong_ver_run_dir / "phenix" / "cache").exists())
+ wrong_ver_preflight_path = (
+ wrong_ver_run_dir
+ / "phenix"
+ / "phenix_preflight_build_evidence.json"
+ )
+ wrong_ver_failure_path = (
+ wrong_ver_run_dir
+ / "phenix"
+ / "phenix_build_admission_failure.json"
+ )
+ self.assertTrue(wrong_ver_preflight_path.is_file())
+ self.assertTrue(wrong_ver_failure_path.is_file())
+ self.assertEqual(
+ json.loads(wrong_ver_preflight_path.read_text(encoding="utf-8")),
+ wrong_ver_build,
+ )
+ wrong_ver_failure_doc = json.loads(
+ wrong_ver_failure_path.read_text(encoding="utf-8")
+ )
+ self.assertEqual(wrong_ver_failure_doc["status"], "rejected")
+ self.assertFalse(wrong_ver_failure_doc["worker_launched"])
+ self.assertEqual(
+ wrong_ver_failure_doc["measured_build"], wrong_ver_build
+ )
+ self.assertEqual(
+ wrong_ver_failure_doc["failure"]["failed_stage"],
+ "validate_measured_phenix_build",
+ )
+
+ def fake_success_runner(
+ argv: list[str], log: Path, *, cwd: Path, timeout: float
+ ) -> subprocess.CompletedProcess[str]:
+ self.assertEqual(
+ Path(argv[0]),
+ (fake_bin / "phenix.python").resolve(strict=True),
+ )
+ mtz_arg = Path(argv[argv.index("--mtz") + 1])
+ out_json = Path(argv[argv.index("--out-json") + 1])
+ out_maps = Path(argv[argv.index("--out-maps-mtz") + 1])
+ out_obs_pair = Path(argv[argv.index("--out-obs-pair-mtz") + 1])
+ out_diag = Path(
+ argv[argv.index("--out-diagnostics-json") + 1]
+ )
+ # Outputs must be placed inside the cache attempt directory (cwd).
+ self.assertEqual(out_json.parent, cwd)
+ self.assertEqual(out_maps.parent, cwd)
+ self.assertEqual(out_obs_pair.parent, cwd)
+ self.assertEqual(out_diag.parent, cwd)
+
+ ledger = onesar_rerun.build_input_hkl_ledger(
+ mtz_arg.read_bytes()
+ )
+ work_candidates = [
+ v
+ for v in ledger["eligible_by_hkl"].values()
+ if not v["is_free"] and v["f_obs"] > 0.0
+ ][:2]
+ free_candidates = [
+ v
+ for v in ledger["eligible_by_hkl"].values()
+ if v["is_free"] and v["f_obs"] > 0.0
+ ][:2]
+ w0, w1 = work_candidates
+ f0, f1 = free_candidates
+
+ scale_k1_w = 0.5
+ scale_k1_t = 0.5
+ scale_k1 = 0.5
+ _, base_payload, _, _, _ = _make_reconciliation_fixture()
+ synth = copy.deepcopy(base_payload)
+ synth["expected_resolved_prefix"] = resolved_prefix_str
+ synth["expected_phenix_version"] = toolchain.PHENIX_VERSION
+ synth["runtime_environment"]["sys_prefix"] = str(fake_prefix)
+ synth["runtime_environment"]["resolved_sys_prefix"] = (
+ resolved_prefix_str
+ )
+ synth["runtime_environment"]["sys_executable"] = str(
+ (fake_bin / "phenix.python").resolve(strict=True)
+ )
+ synth["runtime_environment"]["resolved_sys_executable"] = str(
+ (fake_bin / "phenix.python").resolve(strict=True)
+ )
+ synth["runtime_environment"]["vendor_modules"] = (
+ vendor_modules_payload
+ )
+ selected = [w0, w1, f0, f1]
+ synth["extracted_pre_fmodel"] = [
+ {
+ "hkl": list(item["hkl"]),
+ "f_obs": float(item["f_obs"]),
+ "sigf_obs": float(item["sigf_obs"]),
+ "is_free": bool(item["is_free"]),
+ }
+ for item in selected
+ ]
+ synth["final_effective"] = copy.deepcopy(
+ synth["extracted_pre_fmodel"]
+ )
+ synth["actual_work"] = [
+ {
+ "hkl": list(item["hkl"]),
+ "f_obs": float(item["f_obs"]),
+ "sigf_obs": float(item["sigf_obs"]),
+ "f_model_real": float(item["f_obs"]) / scale_k1_w,
+ "f_model_imag": 0.0,
+ "f_model_amplitude": float(item["f_obs"]) / scale_k1_w,
+ "f_model_scaled_with_k1_w_real": float(item["f_obs"]),
+ "f_model_scaled_with_k1_w_imag": 0.0,
+ "f_model_scaled_with_k1_w_amplitude": float(
+ item["f_obs"]
+ ),
+ }
+ for item in (w0, w1)
+ ]
+ synth["actual_free"] = [
+ {
+ "hkl": list(item["hkl"]),
+ "f_obs": float(item["f_obs"]),
+ "sigf_obs": float(item["sigf_obs"]),
+ "f_model_real": float(item["f_obs"]) / scale_k1_t,
+ "f_model_imag": 0.0,
+ "f_model_amplitude": float(item["f_obs"]) / scale_k1_t,
+ "f_model_scaled_with_k1_t_real": float(item["f_obs"]),
+ "f_model_scaled_with_k1_t_imag": 0.0,
+ "f_model_scaled_with_k1_t_amplitude": float(
+ item["f_obs"]
+ ),
+ }
+ for item in (f0, f1)
+ ]
+ w_den = float(w0["f_obs"] + w1["f_obs"])
+ f_den = float(f0["f_obs"] + f1["f_obs"])
+ synth["scales_and_rfactors"].update(
+ {
+ "r_work": 0.0,
+ "r_free": 0.0,
+ "r_all": 0.0,
+ "r_free_r_work_gap": 0.0,
+ "scale_k1_w": scale_k1_w,
+ "scale_k1_t": scale_k1_t,
+ "scale_k1": scale_k1,
+ "work_numerator": 0.0,
+ "work_denominator": w_den,
+ "free_numerator": 0.0,
+ "free_denominator": f_den,
+ "all_numerator": 0.0,
+ "all_denominator": w_den + f_den,
+ "work_count": 2,
+ "free_count": 2,
+ "final_effective_count": 4,
+ "extracted_pre_fmodel_count": 4,
+ }
+ )
+ map_recs = [
+ {
+ "hkl": list(item["hkl"]),
+ "real": 1.0,
+ "imag": 0.0,
+ "amplitude": 1.0,
+ "phase_deg": 0.0,
+ "is_centric": True,
+ }
+ for item in selected
+ ]
+ synth["map_coefficients"]["2mFo-DFc"]["records"] = copy.deepcopy(
+ map_recs
+ )
+ synth["map_coefficients"]["2mFo-DFc"]["centric_count"] = 4
+ synth["map_coefficients"]["2mFo-DFc"]["acentric_count"] = 0
+ synth["map_coefficients"]["mFo-DFc"]["records"] = copy.deepcopy(
+ map_recs
+ )
+ synth["map_coefficients"]["mFo-DFc"]["centric_count"] = 4
+ synth["map_coefficients"]["mFo-DFc"]["acentric_count"] = 0
+
+ cell_floats = tuple(
+ float(x)
+ for x in ledger["crystallographic_context"][
+ "cell_a_b_c_alpha_beta_gamma"
+ ]
+ )
+ runtime_cell_floats = onesar_rerun.ccp4_mtz_float32_unit_cell(
+ ledger["crystallographic_context"][
+ "cell_a_b_c_alpha_beta_gamma"
+ ]
+ )
+ synth["crystal_symmetry"] = {
+ "unit_cell": list(runtime_cell_floats),
+ "space_group_symbol": "P 21 21 21 (No. 19)",
+ "space_group_number": 19,
+ "space_group_operators_xyz": [
+ "x,y,z",
+ "-x+1/2,-y,z+1/2",
+ "-x,y+1/2,-z+1/2",
+ "x+1/2,-y+1/2,-z",
+ ],
+ }
+ ca, cb, cc = (
+ runtime_cell_floats[0],
+ runtime_cell_floats[1],
+ runtime_cell_floats[2],
+ )
+
+ def _orth_range(
+ items: list[dict[str, Any]],
+ ) -> dict[str, float]:
+ ds = [
+ 1.0
+ / math.sqrt(
+ (it["hkl"][0] / ca) ** 2
+ + (it["hkl"][1] / cb) ** 2
+ + (it["hkl"][2] / cc) ** 2
+ )
+ for it in items
+ ]
+ return {"d_max": max(ds), "d_min": min(ds)}
+
+ synth["resolution_ranges"] = {
+ "extracted_pre_fmodel": _orth_range(selected),
+ "final_effective": _orth_range(selected),
+ "actual_work": _orth_range([w0, w1]),
+ "actual_free": _orth_range([f0, f1]),
+ }
+ m_bytes, p_bytes = _build_synthetic_map_mtz_pair(
+ map_recs, map_recs, cell=cell_floats # type: ignore[arg-type]
+ )
+ synth["map_coefficients"]["mtz_supplements"][
+ "phx_maps_sha256"
+ ] = hashlib.sha256(m_bytes).hexdigest()
+ synth["map_coefficients"]["mtz_supplements"][
+ "obs_pair_only_sha256"
+ ] = hashlib.sha256(p_bytes).hexdigest()
+
+ diag_doc = {
+ "schema_version": 1,
+ "status": "succeeded",
+ "extract_xtal_data_log": synth["diagnostics"][
+ "extract_xtal_data_log"
+ ],
+ "model_manager_log": synth["diagnostics"][
+ "model_manager_log"
+ ],
+ "fmodel_update_all_scales_log": synth["diagnostics"][
+ "fmodel_update_all_scales_log"
+ ],
+ "failure": None,
+ }
+ log.write_text("synthetic phenix.python stdout\n", encoding="utf-8")
+ out_maps.write_bytes(m_bytes)
+ out_obs_pair.write_bytes(p_bytes)
+ out_diag.write_text(
+ json.dumps(diag_doc, indent=2, sort_keys=True) + "\n",
+ encoding="utf-8",
+ )
+ out_json.write_text(
+ json.dumps(synth, indent=2, sort_keys=True) + "\n",
+ encoding="utf-8",
+ )
+ return subprocess.CompletedProcess(argv, 0)
+
+ with (
+ mock.patch.object(
+ toolchain,
+ "phenix",
+ side_effect=lambda exe: fake_bin / exe,
+ ),
+ mock.patch.object(
+ toolchain,
+ "phenix_build_evidence",
+ return_value=fake_measured_build,
+ ),
+ ):
+ reconciled = onesar_rerun.run_phenix_rfactor_operation(
+ fake_repo / "runs" / "r4_success",
+ subject_id="R4",
+ repo_root=fake_repo,
+ archive_path=COMMITTED_ARCHIVE,
+ require_no_protstruct_overrides=False,
+ runner=fake_success_runner,
+ )
+
+ self.assertEqual(
+ reconciled["operation_manifest"]["returncode"], 0
+ )
+ self.assertTrue(reconciled["operation_manifest"]["reusable"])
+ manifest_path = Path(reconciled["manifest_path"])
+ self.assertTrue(manifest_path.is_file())
+ self.assertEqual(manifest_path.name, "success.json")
+ self.assertEqual(manifest_path.parent.name, "bundle")
+ self.assertTrue(
+ Path(reconciled["phenix_rfactors_json_path"]).is_file()
+ )
+ self.assertTrue(
+ Path(reconciled["phenix_diagnostics_json_path"]).is_file()
+ )
+ self.assertTrue(
+ Path(reconciled["input_hkl_ledger_json_path"]).is_file()
+ )
+ self.assertTrue(Path(reconciled["phx_maps_mtz_path"]).is_file())
+ self.assertTrue(
+ Path(reconciled["phx_obs_pair_only_mtz_path"]).is_file()
+ )
+ self.assertTrue(
+ Path(
+ reconciled["preflight_build_evidence_json_path"]
+ ).is_file()
+ )
+ self.assertEqual(
+ json.loads(
+ Path(
+ reconciled["preflight_build_evidence_json_path"]
+ ).read_text(encoding="utf-8")
+ ),
+ fake_measured_build,
+ )
+ self.assertFalse(
+ (
+ fake_repo
+ / "runs"
+ / "r4_success"
+ / "phenix"
+ / "phenix_build_admission_failure.json"
+ ).exists()
+ )
+ self.assertEqual(
+ reconciled["reconciliation"]["reconciliation_status"],
+ "verified",
+ )
+
+ # Failure retention test: non-zero exit raises OperationFailed and retains log
+ # as well as parent failure manifest with paths/hashes of partial/diagnostic artifacts (#1014).
+ def fake_failing_runner(
+ argv: list[str], log: Path, *, cwd: Path, timeout: float
+ ) -> subprocess.CompletedProcess[str]:
+ out_diag = Path(
+ argv[argv.index("--out-diagnostics-json") + 1]
+ )
+ out_partial = Path(
+ argv[argv.index("--out-partial-json") + 1]
+ )
+ self.assertEqual(out_diag.parent, cwd)
+ self.assertEqual(out_partial.parent, cwd)
+ log.write_text(
+ "simulated extraction failure\n", encoding="utf-8"
+ )
+ onesar_phenix_measure._atomic_write_json_checkpoint(
+ {
+ "schema_version": 1,
+ "status": "failed",
+ "current_phase": "rfactor_replay",
+ "completed_raw_stages": [
+ "extract_xtal_data",
+ "xray_structure_invariance",
+ "rfactor_raw_arrays_and_scalars",
+ ],
+ "partial_evidence_status": "failed_partial_unverified",
+ "failure": {
+ "failed_phase": "rfactor_replay",
+ "exception_type": "RuntimeError",
+ "exception_message": "simulated rfactor mismatch",
+ },
+ },
+ out_diag,
+ )
+ onesar_phenix_measure._atomic_write_json_checkpoint(
+ {
+ "schema_version": 1,
+ "artifact_role": "partial_unverified_phenix_evidence",
+ "reusable_cache_output": False,
+ "canonical_eval_qds_eligible": False,
+ "status": "failed_partial_unverified",
+ "current_phase": "rfactor_replay",
+ "completed_stages": [
+ "extract_xtal_data",
+ "xray_structure_invariance",
+ "rfactor_raw_arrays_and_scalars",
+ ],
+ "rfactor_replay_verified": False,
+ "map_export_verified": False,
+ "failure": {
+ "failed_phase": "rfactor_replay",
+ "exception_type": "RuntimeError",
+ "exception_message": "simulated rfactor mismatch",
+ },
+ },
+ out_partial,
+ )
+ return subprocess.CompletedProcess(argv, 2)
+
+ r4_failure_dir = fake_repo / "runs" / "r4_failure"
+ with self.assertRaises(refinement_cache.OperationFailed) as ctx:
+ onesar_rerun.run_phenix_rfactor_operation(
+ r4_failure_dir,
+ subject_id="R4",
+ repo_root=fake_repo,
+ archive_path=COMMITTED_ARCHIVE,
+ require_no_protstruct_overrides=False,
+ runner=fake_failing_runner,
+ )
+ self.assertTrue(ctx.exception.log_path.is_file())
+ self.assertIn(
+ "simulated extraction failure",
+ ctx.exception.log_path.read_text(encoding="utf-8"),
+ )
+ parent_failure_path = (
+ r4_failure_dir / "phenix" / "phenix_operation_failure.json"
+ )
+ self.assertTrue(parent_failure_path.is_file())
+ parent_failure_doc = json.loads(
+ parent_failure_path.read_text(encoding="utf-8")
+ )
+ self.assertEqual(parent_failure_doc["status"], "failed")
+ self.assertFalse(parent_failure_doc["reusable_cache_output"])
+ self.assertFalse(
+ parent_failure_doc["canonical_eval_qds_eligible"]
+ )
+ self.assertIn(
+ "phenix_partial_evidence_json",
+ parent_failure_doc["retained_attempt_artifacts"],
+ )
+ self.assertIn(
+ "phenix_diagnostics_json",
+ parent_failure_doc["retained_attempt_artifacts"],
+ )
+ retained_partial_entry = parent_failure_doc[
+ "retained_attempt_artifacts"
+ ]["phenix_partial_evidence_json"]
+ self.assertEqual(
+ retained_partial_entry["sha256"],
+ hashlib.sha256(
+ Path(retained_partial_entry["path"]).read_bytes()
+ ).hexdigest(),
+ )
+ self.assertEqual(
+ parent_failure_doc["partial_evidence_summary"][
+ "completed_stages"
+ ],
+ [
+ "extract_xtal_data",
+ "xray_structure_invariance",
+ "rfactor_raw_arrays_and_scalars",
+ ],
+ )
+ self.assertFalse(
+ (
+ r4_failure_dir
+ / "phenix"
+ / "phenix_reconciled_manifest.json"
+ ).exists()
+ )
+
+ # Drift between preflight phenix_build_evidence() and cached_phenix_operation's
+ # internal phenix_build_evidence() call is rejected (#1013).
+ drifted_build = copy.deepcopy(fake_measured_build)
+ drifted_build["version_probe"]["setup_sha256"] = "b" * 64
+ with (
+ mock.patch.object(
+ toolchain,
+ "phenix",
+ side_effect=lambda exe: fake_bin / exe,
+ ),
+ mock.patch.object(
+ toolchain,
+ "phenix_build_evidence",
+ side_effect=[fake_measured_build, drifted_build],
+ ),
+ ):
+ with self.assertRaisesRegex(
+ ValueError,
+ "Cache manifest measured_build drifted from preflight",
+ ):
+ onesar_rerun.run_phenix_rfactor_operation(
+ fake_repo / "runs" / "r4_drift",
+ subject_id="R4",
+ repo_root=fake_repo,
+ archive_path=COMMITTED_ARCHIVE,
+ require_no_protstruct_overrides=False,
+ runner=fake_success_runner,
+ )
+
+ # Hermetic local-Python subprocess timeout test (#1014):
+ # Invoke the actual onesar_phenix_measure.py worker in a child subprocess via
+ # cached_phenix_operation + toolchain.run_logged (runner=None) with a bounded
+ # timeout while fmodel.update_all_scales writes through _PersistentTeeLogStream
+ # and then blocks past the timeout (so finally: and except: never run).
+ timeout_prefix = tmp_root / "phenix-timeout-2.0-5936"
+ timeout_bin = timeout_prefix / "phenix_bin"
+ timeout_site = (
+ timeout_prefix / "lib" / "python3.9" / "site-packages"
+ )
+ timeout_bin.mkdir(parents=True)
+ (timeout_prefix / "phenix_env.sh").write_text(
+ "#!/bin/sh\nexport PHENIX_VERSION=2.0\nexport PHENIX_RELEASE_TAG=5936\n",
+ encoding="utf-8",
+ )
+ (timeout_bin / "phenix.version").write_text(
+ "#!/bin/sh\n", encoding="utf-8"
+ )
+ for pkg_rel in (
+ "cctbx/__init__.py",
+ "iotbx/__init__.py",
+ "iotbx/mtz/__init__.py",
+ "mmtbx/__init__.py",
+ "mmtbx/f_model/__init__.py",
+ ):
+ pkg_init = timeout_site / pkg_rel
+ pkg_init.parent.mkdir(parents=True, exist_ok=True)
+ pkg_init.write_text(
+ "from .f_model import manager\n"
+ if pkg_rel == "mmtbx/f_model/__init__.py"
+ else "# synthetic package\n",
+ encoding="utf-8",
+ )
+
+ (timeout_site / "cctbx" / "crystal.py").write_text(
+ "import struct, types\n"
+ "_F32_CELL = struct.unpack('<6f', struct.pack('<6f', 64.897, 78.323, 38.792, 90.0, 90.0, 90.0))\n"
+ "_OPS = [types.SimpleNamespace(as_xyz=lambda s=op: s) for op in ('x,y,z', '-x+1/2,-y,z+1/2', '-x,y+1/2,-z+1/2', 'x+1/2,-y+1/2,-z')]\n"
+ "_SG = types.SimpleNamespace(type=lambda: types.SimpleNamespace(number=lambda: 19), all_ops=lambda: list(_OPS))\n"
+ "_SGI = types.SimpleNamespace(symbol_and_number=lambda: 'P 21 21 21 (No. 19)', type=lambda: types.SimpleNamespace(number=lambda: 19), group=lambda: _SG)\n"
+ "_SYM = types.SimpleNamespace(\n"
+ " unit_cell=lambda: types.SimpleNamespace(parameters=lambda: _F32_CELL),\n"
+ " space_group=lambda: _SG,\n"
+ " space_group_info=lambda: _SGI,\n"
+ ")\n"
+ "def select_crystal_symmetry(**kwargs):\n"
+ " return _SYM\n",
+ encoding="utf-8",
+ )
+ (timeout_site / "iotbx" / "pdb.py").write_text(
+ "from cctbx.crystal import _SYM\n"
+ "class _Inp:\n"
+ " def crystal_symmetry(self):\n"
+ " return _SYM\n"
+ "def input(file_name):\n"
+ " return _Inp()\n",
+ encoding="utf-8",
+ )
+ (timeout_site / "iotbx" / "mtz" / "__init__.py").write_text(
+ "import types\n"
+ "def label_decorator(phases_prefix='PH'):\n"
+ " return types.SimpleNamespace(phases_prefix=phases_prefix)\n",
+ encoding="utf-8",
+ )
+ (timeout_site / "iotbx" / "reflection_file_reader.py").write_text(
+ "import types\n"
+ "from cctbx.crystal import _SYM\n"
+ "def any_reflection_file(file_name):\n"
+ " return types.SimpleNamespace(\n"
+ " file_type=lambda: 'ccp4_mtz',\n"
+ " as_miller_arrays=lambda: [types.SimpleNamespace(crystal_symmetry=lambda: _SYM)],\n"
+ " )\n",
+ encoding="utf-8",
+ )
+ (timeout_site / "iotbx" / "reflection_file_utils.py").write_text(
+ "def reflection_file_server(**kwargs):\n return 'RFS'\n",
+ encoding="utf-8",
+ )
+ (timeout_site / "iotbx" / "extract_xtal_data.py").write_text(
+ "import io, struct, types\n"
+ "_CA_F32 = struct.unpack(' subprocess.CompletedProcess[str]:
+ cwd_path = Path(kwargs["cwd"])
+ diag_file = cwd_path / "phenix_diagnostics.json"
+ proc = subprocess.Popen(
+ cmd,
+ cwd=kwargs.get("cwd"),
+ stdout=kwargs.get("stdout"),
+ stderr=kwargs.get("stderr"),
+ text=kwargs.get("text", True),
+ env=kwargs.get("env"),
+ )
+ try:
+ deadline = time.monotonic() + 15.0
+ while time.monotonic() < deadline:
+ if diag_file.is_file():
+ try:
+ doc = json.loads(
+ diag_file.read_text(encoding="utf-8")
+ )
+ if (
+ doc.get("current_phase")
+ == "update_all_scales"
+ and "LIVE_SCALING_MARKER_BEFORE_TIMEOUT"
+ in (
+ doc.get("fmodel_update_all_scales_log")
+ or ""
+ )
+ ):
+ break
+ except (OSError, ValueError):
+ pass
+ if proc.poll() is not None:
+ break
+ time.sleep(0.01)
+ proc.communicate(timeout=0.05)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+ proc.wait()
+ raise
+ return subprocess.CompletedProcess(cmd, proc.returncode or 0)
+
+ timeout_run_dir = fake_repo / "runs" / "r4_live_timeout"
+ with (
+ mock.patch.object(
+ toolchain,
+ "phenix",
+ side_effect=lambda exe: timeout_bin / exe,
+ ),
+ mock.patch.object(
+ toolchain,
+ "phenix_build_evidence",
+ return_value=timeout_measured_build,
+ ),
+ mock.patch.object(
+ toolchain.subprocess,
+ "run",
+ side_effect=_readiness_aware_timeout_subprocess_run,
+ ),
+ ):
+ with self.assertRaises(refinement_cache.OperationFailed) as t_ctx:
+ onesar_rerun.run_phenix_rfactor_operation(
+ timeout_run_dir,
+ subject_id="R4",
+ repo_root=fake_repo,
+ archive_path=COMMITTED_ARCHIVE,
+ require_no_protstruct_overrides=False,
+ runner=None,
+ timeout=2.5,
+ )
+
+ self.assertTrue(t_ctx.exception.log_path.is_file())
+ self.assertIn(
+ "PHENIX operation timed out; retained log:",
+ str(t_ctx.exception),
+ )
+ self.assertIsInstance(
+ t_ctx.exception.__cause__, subprocess.TimeoutExpired
+ )
+ stdout_log_text = t_ctx.exception.log_path.read_text(
+ encoding="utf-8"
+ )
+ self.assertIn(
+ "=== [onesar_phenix_measure] mmtbx.model.manager ===",
+ stdout_log_text,
+ )
+ self.assertIn(
+ "LIVE_MODEL_MANAGER_MARKER_BEFORE_SCALES",
+ stdout_log_text,
+ )
+ self.assertIn(
+ "=== [onesar_phenix_measure] fmodel.update_all_scales ===",
+ stdout_log_text,
+ )
+ self.assertIn(
+ "LIVE_SCALING_MARKER_BEFORE_TIMEOUT",
+ stdout_log_text,
+ )
+
+ timeout_diag_path = (
+ t_ctx.exception.log_path.parent / "phenix_diagnostics.json"
+ )
+ self.assertTrue(timeout_diag_path.is_file())
+ timeout_diag_doc = json.loads(
+ timeout_diag_path.read_text(encoding="utf-8")
+ )
+ self.assertEqual(timeout_diag_doc["status"], "running")
+ self.assertEqual(
+ timeout_diag_doc["current_phase"], "update_all_scales"
+ )
+ self.assertEqual(
+ timeout_diag_doc["phase_history"],
+ [
+ "parameter_validation",
+ "vendor_imports",
+ "runtime_environment_inspection",
+ "pdb_input",
+ "reflection_file_input",
+ "crystal_symmetry_consensus",
+ "extract_xtal_data",
+ "model_manager",
+ "fmodel_manager_init",
+ "update_all_scales",
+ ],
+ )
+ self.assertEqual(
+ timeout_diag_doc["extract_xtal_data_log"],
+ "LIVE_EXTRACT_XTAL_DATA_MARKER\n",
+ )
+ self.assertEqual(
+ timeout_diag_doc["model_manager_log"],
+ "LIVE_MODEL_MANAGER_MARKER_BEFORE_SCALES\n",
+ )
+ self.assertEqual(
+ timeout_diag_doc["fmodel_update_all_scales_log"],
+ "LIVE_SCALING_MARKER_BEFORE_TIMEOUT\n",
+ )
+ self.assertIsNone(timeout_diag_doc["failure"])
+
+ timeout_partial_path = (
+ t_ctx.exception.log_path.parent
+ / "phenix_rfactors_partial_evidence.json"
+ )
+ self.assertTrue(timeout_partial_path.is_file())
+ timeout_partial_doc = json.loads(
+ timeout_partial_path.read_text(encoding="utf-8")
+ )
+ self.assertEqual(
+ timeout_partial_doc["status"], "running_partial_unverified"
+ )
+ self.assertEqual(
+ timeout_partial_doc["completed_stages"],
+ ["extract_xtal_data", "xray_structure_snapshot_before"],
+ )
+ self.assertEqual(
+ len(timeout_partial_doc["extracted_pre_fmodel"]), 2
+ )
+ self.assertEqual(
+ timeout_partial_doc["xray_structure_snapshots_raw"]["before"][
+ "n_scatterers"
+ ],
+ 1,
+ )
+ self.assertIsNone(
+ timeout_partial_doc["xray_structure_snapshots_raw"][
+ "after_model"
+ ]
+ )
+ self.assertIsNone(
+ timeout_partial_doc["xray_structure_snapshots_raw"][
+ "after_fmodel"
+ ]
+ )
+ self.assertFalse(
+ timeout_partial_doc["xray_structure_snapshots_raw"][
+ "invariance_verified"
+ ]
+ )
+ self.assertIsNone(timeout_partial_doc["xray_structure_invariance"])
+ timeout_parent_fail = (
+ timeout_run_dir / "phenix" / "phenix_operation_failure.json"
+ )
+ self.assertTrue(timeout_parent_fail.is_file())
+ timeout_parent_doc = json.loads(
+ timeout_parent_fail.read_text(encoding="utf-8")
+ )
+ self.assertIn(
+ "phenix_partial_evidence_json",
+ timeout_parent_doc["retained_attempt_artifacts"],
+ )
+ self.assertIn(
+ "phenix_diagnostics_json",
+ timeout_parent_doc["retained_attempt_artifacts"],
+ )
+
+ def test_in_process_measure_first_block_rfactors_with_source_backed_doubles_1017(
+ self,
+ ) -> None:
+ """Directly execute measure_first_block_rfactors in-process with narrow source-backed doubles."""
+
+ class _FakeArray(list):
+ def size(self) -> int:
+ return len(self)
+
+ def all_eq(self, other: Any) -> bool:
+ return list(self) == list(other)
+
+ class _FakeMillerArray:
+ def __init__(
+ self,
+ indices: list[tuple[int, int, int]],
+ data: list[Any],
+ sigmas: list[float] | None = None,
+ centric_flags: list[bool] | None = None,
+ ) -> None:
+ self._indices = _FakeArray(indices)
+ self._data = _FakeArray(data)
+ self._sigmas = (
+ _FakeArray(sigmas) if sigmas is not None else None
+ )
+ self._centric = (
+ _FakeArray(centric_flags)
+ if centric_flags is not None
+ else _FakeArray([idx[1] == 0 for idx in indices])
+ )
+
+ def indices(self) -> _FakeArray:
+ return self._indices
+
+ def data(self) -> _FakeArray:
+ return self._data
+
+ def sigmas(self) -> _FakeArray | None:
+ return self._sigmas
+
+ def size(self) -> int:
+ return len(self._indices)
+
+ def _d_values(self) -> list[float]:
+ f32_cell = onesar_rerun.ccp4_mtz_float32_unit_cell(
+ (64.897, 78.323, 38.792, 90.0, 90.0, 90.0)
+ )
+ return [
+ onesar_rerun.compute_hkl_d_spacing((h, k, l), f32_cell)
+ for h, k, l in self._indices
+ ]
+
+ def d_min(self) -> float:
+ return min(self._d_values())
+
+ def d_max_min(self) -> tuple[float, float]:
+ d_vals = self._d_values()
+ return (max(d_vals), min(d_vals))
+
+ def info(self) -> str:
+ return "synthetic_miller_array"
+
+ def centric_flags(self) -> Any:
+ parent = self
+
+ class _Flags:
+ def data(self_inner) -> _FakeArray:
+ return parent._centric
+
+ return _Flags()
+
+ def as_mtz_dataset(
+ self, column_root_label: str, label_decorator: Any = None
+ ) -> Any:
+ return _FakeMtzDataset(self, column_root_label)
+
+ class _FakeMtzDataset:
+ fail_in_map_write: bool = False
+
+ def __init__(self, first_array: _FakeMillerArray, root_label: str) -> None:
+ self.arrays: list[tuple[_FakeMillerArray, str]] = [
+ (first_array, root_label)
+ ]
+
+ def add_miller_array(
+ self,
+ miller_array: _FakeMillerArray,
+ column_root_label: str,
+ label_decorator: Any = None,
+ ) -> None:
+ self.arrays.append((miller_array, column_root_label))
+
+ def mtz_object(self) -> Any:
+ parent = self
+
+ class _Writer:
+ def column_labels(self_inner) -> list[str]:
+ cols = ["H", "K", "L"]
+ for _, lbl in parent.arrays:
+ cols.extend([lbl, f"PH{lbl}"])
+ return cols
+
+ def write(self_inner, file_name: str) -> None:
+ if _FakeMtzDataset.fail_in_map_write:
+ raise RuntimeError(
+ "Injected synthetic map write failure"
+ )
+ recs_by_label: dict[str, list[dict[str, Any]]] = {}
+ for arr, lbl in parent.arrays:
+ recs: list[dict[str, Any]] = []
+ for idx, cval, is_c in zip(
+ arr.indices(),
+ arr.data(),
+ arr.centric_flags().data(),
+ ):
+ re_v = float(cval.real)
+ im_v = float(cval.imag)
+ amp_v = math.hypot(re_v, im_v)
+ ph_v = (
+ 0.0
+ if amp_v == 0.0
+ else math.degrees(math.atan2(im_v, re_v))
+ )
+ recs.append(
+ {
+ "hkl": list(idx),
+ "real": re_v,
+ "imag": im_v,
+ "amplitude": amp_v,
+ "phase_deg": ph_v,
+ "is_centric": bool(is_c),
+ }
+ )
+ recs_by_label[lbl] = recs
+ if "FOFCWT" in recs_by_label:
+ m_bytes, _ = _build_synthetic_map_mtz_pair(
+ recs_by_label["2FOFCWT"],
+ recs_by_label["FOFCWT"],
+ )
+ Path(file_name).write_bytes(m_bytes)
+ else:
+ _, p_bytes = _build_synthetic_map_mtz_pair(
+ recs_by_label["2FOFCWT"],
+ recs_by_label["2FOFCWT"],
+ )
+ Path(file_name).write_bytes(p_bytes)
+
+ return _Writer()
+
+ class _FakeMtzObjectReadback:
+ def __init__(self, file_name: str) -> None:
+ mtz_bytes = Path(file_name).read_bytes()
+ _, headers = onesar_rerun._mtz_headers(mtz_bytes)
+ ncol_row = [
+ line.split() for line in headers if line.startswith("NCOL ")
+ ][0]
+ self._nrows = int(ncol_row[2])
+ col_rows = [
+ line.split()
+ for line in headers
+ if line.startswith("COLUMN ")
+ ]
+ self._labels = [parts[1] for parts in col_rows]
+ self._types = [parts[2] for parts in col_rows]
+
+ def columns(self) -> list[Any]:
+ return [
+ types.SimpleNamespace(
+ label=lambda l=lbl: l,
+ type=lambda t=tp: t,
+ )
+ for lbl, tp in zip(self._labels, self._types)
+ ]
+
+ def column_labels(self) -> list[str]:
+ return list(self._labels)
+
+ def column_types(self) -> list[str]:
+ return list(self._types)
+
+ def n_reflections(self) -> int:
+ return self._nrows
+
+ class _FakeScatterer:
+ def __init__(self) -> None:
+ self.label = "PDB=\" N VAL A 1 \""
+ self.scattering_type = "N"
+ self.site = (0.1, 0.2, 0.3)
+ self.occupancy = 1.0
+ self.u_iso = 0.25
+ self.u_star = (0.0, 0.0, 0.0, 0.0, 0.0, 0.0)
+ self.flags = types.SimpleNamespace(
+ use_u_iso=lambda: True,
+ use_u_aniso=lambda: False,
+ )
+
+ class _FakeXrayStructure:
+ def scatterers(self) -> list[_FakeScatterer]:
+ return [_FakeScatterer()]
+
+ def sites_cart(self) -> list[tuple[float, float, float]]:
+ return [(6.4897, 15.6646, 11.6376)]
+
+ def scattering_type_registry(self) -> Any:
+ return types.SimpleNamespace(
+ Unassigned=False,
+ has_unknown_type_symbols=lambda: False,
+ )
+
+ hkls_5 = [(1, 0, 0), (0, 1, 0), (1, 1, 1), (2, 0, 0), (0, 2, 0)]
+ f_obs_5 = [100.0, 200.0, 150.0, 80.0, 120.0]
+ sigf_5 = [2.0, 3.0, 2.5, 1.8, 2.2]
+ flags_5 = [False, False, False, True, True]
+
+ hkls_4 = [(1, 0, 0), (0, 1, 0), (2, 0, 0), (0, 2, 0)]
+ f_obs_4 = [100.0, 200.0, 80.0, 120.0]
+ sigf_4 = [2.0, 3.0, 1.8, 2.2]
+ flags_4 = [False, False, True, True]
+
+ hkls_w = [(1, 0, 0), (0, 1, 0)]
+ f_obs_w = [100.0, 200.0]
+ sigf_w = [2.0, 3.0]
+ fm_w = [complex(108.0, 144.0), complex(252.0, 336.0)]
+ fm_scaled_w = [complex(54.0, 72.0), complex(126.0, 168.0)]
+
+ hkls_t = [(2, 0, 0), (0, 2, 0)]
+ f_obs_t = [80.0, 120.0]
+ sigf_t = [1.8, 2.2]
+ fm_t = [complex(60.0, 80.0), complex(150.0, 200.0)]
+ fm_scaled_t = [complex(36.0, 48.0), complex(90.0, 120.0)]
+
+ _fake_f32_cell = onesar_rerun.ccp4_mtz_float32_unit_cell(
+ (64.897, 78.323, 38.792, 90.0, 90.0, 90.0)
+ )
+ _fake_rt_mx_ops = [
+ types.SimpleNamespace(as_xyz=lambda s=op: s)
+ for op in (
+ "x,y,z",
+ "-x+1/2,-y,z+1/2",
+ "-x,y+1/2,-z+1/2",
+ "x+1/2,-y+1/2,-z",
+ )
+ ]
+ _fake_space_group = types.SimpleNamespace(
+ type=lambda: types.SimpleNamespace(number=lambda: 19),
+ all_ops=lambda: list(_fake_rt_mx_ops),
+ )
+ _fake_space_group_info = types.SimpleNamespace(
+ symbol_and_number=lambda: "P 21 21 21 (No. 19)",
+ type=lambda: types.SimpleNamespace(number=lambda: 19),
+ group=lambda: _fake_space_group,
+ )
+ _fake_symmetry = types.SimpleNamespace(
+ unit_cell=lambda: types.SimpleNamespace(
+ parameters=lambda: _fake_f32_cell
+ ),
+ space_group=lambda: _fake_space_group,
+ space_group_info=lambda: _fake_space_group_info,
+ )
+
+ def _make_vendor_modules(
+ *,
+ df_err: list[str] | None = None,
+ raise_in_extract: bool = False,
+ fail_in_fmodel: bool = False,
+ fail_in_map_write: bool = False,
+ fail_in_map_coefficients: bool = False,
+ fail_in_second_map_coefficients: bool = False,
+ mutate_scatterer_count_in_scales: bool = False,
+ mutate_scatterer_occupancy_in_scales: bool = False,
+ nonfinite_scatterer_occupancy_in_scales: float | None = None,
+ nonfinite_scatterer_u_iso_in_scales: float | None = None,
+ empty_scatterers_in_scales: bool = False,
+ symmetry_override: Any = None,
+ nonfinite_work_model_coeff: bool = False,
+ nonpositive_final_obs: bool = False,
+ fail_in_free_array: bool = False,
+ native_r_work_override: float | None = None,
+ ) -> dict[str, Any]:
+ _FakeMtzDataset.fail_in_map_write = fail_in_map_write
+ scales_completed = [False]
+ active_symmetry = (
+ symmetry_override
+ if symmetry_override is not None
+ else _fake_symmetry
+ )
+
+ class _ScopedXrayStructure(_FakeXrayStructure):
+ def scatterers(self) -> list[_FakeScatterer]:
+ if scales_completed[0] and empty_scatterers_in_scales:
+ return []
+ if (
+ scales_completed[0]
+ and mutate_scatterer_count_in_scales
+ ):
+ return [_FakeScatterer(), _FakeScatterer()]
+ if (
+ scales_completed[0]
+ and mutate_scatterer_occupancy_in_scales
+ ):
+ sc = _FakeScatterer()
+ sc.occupancy = 0.5
+ return [sc]
+ if (
+ scales_completed[0]
+ and nonfinite_scatterer_occupancy_in_scales is not None
+ ):
+ sc = _FakeScatterer()
+ sc.occupancy = nonfinite_scatterer_occupancy_in_scales
+ return [sc]
+ if (
+ scales_completed[0]
+ and nonfinite_scatterer_u_iso_in_scales is not None
+ ):
+ sc = _FakeScatterer()
+ sc.u_iso = nonfinite_scatterer_u_iso_in_scales
+ return [sc]
+ return super().scatterers()
+
+ def sites_cart(self) -> list[tuple[float, float, float]]:
+ if scales_completed[0] and empty_scatterers_in_scales:
+ return []
+ if (
+ scales_completed[0]
+ and mutate_scatterer_count_in_scales
+ ):
+ return [
+ (6.4897, 15.6646, 11.6376),
+ (6.4897, 15.6646, 11.6376),
+ ]
+ return super().sites_cart()
+
+ iotbx_mod = types.ModuleType("iotbx")
+ iotbx_pdb = types.ModuleType("iotbx.pdb")
+ iotbx_mtz = types.ModuleType("iotbx.mtz")
+ iotbx_rfr = types.ModuleType("iotbx.reflection_file_reader")
+ iotbx_rfu = types.ModuleType("iotbx.reflection_file_utils")
+ iotbx_exd = types.ModuleType("iotbx.extract_xtal_data")
+ mmtbx_mod = types.ModuleType("mmtbx")
+ mmtbx_fmodel = types.ModuleType("mmtbx.f_model")
+ mmtbx_fmodel_mod = types.ModuleType("mmtbx.f_model.f_model")
+ mmtbx_map_tools = types.ModuleType("mmtbx.map_tools")
+ mmtbx_model = types.ModuleType("mmtbx.model")
+ mmtbx_mvd = types.ModuleType("mmtbx.model_vs_data")
+ cctbx_mod = types.ModuleType("cctbx")
+ cctbx_crystal = types.ModuleType("cctbx.crystal")
+
+ class _PdbInp:
+ def crystal_symmetry(self) -> Any:
+ return active_symmetry
+
+ def get_r_rfree_sigma(self, *args: Any, **kwargs: Any) -> Any:
+ raise AssertionError(
+ "Second-block get_r_rfree_sigma must never be called"
+ )
+
+ iotbx_pdb.input = lambda file_name: _PdbInp()
+ iotbx_mtz.object = _FakeMtzObjectReadback
+ iotbx_mtz.label_decorator = (
+ lambda phases_prefix="PH": types.SimpleNamespace(
+ phases_prefix=phases_prefix
+ )
+ )
+ iotbx_rfr.any_reflection_file = lambda file_name: types.SimpleNamespace(
+ file_type=lambda: "ccp4_mtz",
+ as_miller_arrays=lambda: [
+ types.SimpleNamespace(
+ crystal_symmetry=lambda: active_symmetry
+ )
+ ],
+ )
+ iotbx_rfu.reflection_file_server = lambda **kwargs: "RFS"
+ mmtbx_mvd.reflection_file_server = lambda **kwargs: "RFS"
+ cctbx_crystal.select_crystal_symmetry = (
+ lambda **kwargs: active_symmetry
+ )
+
+ def _data_and_flags_master_params() -> Any:
+ return types.SimpleNamespace(
+ extract=lambda: types.SimpleNamespace(
+ file_name=None,
+ labels=None,
+ r_free_flags=types.SimpleNamespace(
+ file_name=None,
+ label=None,
+ test_flag_value=None,
+ generate=True,
+ ignore_r_free_flags=True,
+ ),
+ )
+ )
+
+ def _extract_run(
+ reflection_file_server: Any,
+ parameters: Any = None,
+ keep_going: bool = False,
+ ) -> Any:
+ if raise_in_extract:
+ raise RuntimeError(
+ "extract_xtal_data constructor raised before returning"
+ )
+ log_buf = io.StringIO()
+ log_buf.write("Detailed extract_xtal_data log output\n")
+ return types.SimpleNamespace(
+ log=log_buf,
+ err=list(df_err) if df_err is not None else [],
+ f_obs=_FakeMillerArray(hkls_5, f_obs_5, sigf_5),
+ r_free_flags=_FakeMillerArray(hkls_5, flags_5),
+ raw_data=_FakeMillerArray(hkls_5, f_obs_5, sigf_5),
+ raw_flags=_FakeMillerArray(hkls_5, [0, 0, 0, 1, 1]),
+ test_flag_value=1,
+ )
+
+ iotbx_exd.data_and_flags_master_params = (
+ _data_and_flags_master_params
+ )
+ iotbx_exd.run = _extract_run
+
+ def _model_manager(
+ model_input: Any,
+ crystal_symmetry: Any,
+ log: Any,
+ ) -> Any:
+ log.write("model.manager initialization log\n")
+ xrs = _ScopedXrayStructure()
+ return types.SimpleNamespace(
+ get_xray_structure=lambda: xrs,
+ setup_scattering_dictionaries=lambda **kw: None,
+ )
+
+ mmtbx_model.manager = _model_manager
+
+ def _map_coefficients(
+ map_type: str,
+ acentrics_scale: float = 2.0,
+ centrics_pre_scale: float = 1.0,
+ exclude_free_r_reflections: bool = False,
+ fill_missing: bool = False,
+ isotropize: bool = True,
+ sharp: bool = False,
+ ) -> _FakeMillerArray:
+ if fail_in_map_coefficients:
+ raise RuntimeError(
+ "Injected synthetic map_coefficients failure"
+ )
+ if fail_in_second_map_coefficients and map_type == "mFo-DFc":
+ raise RuntimeError(
+ "Injected synthetic second-map coefficient failure"
+ )
+ if map_type == "2mFo-DFc":
+ return _FakeMillerArray(
+ hkls_4,
+ [
+ complex(10.0, 0.0),
+ complex(-12.0, 0.0),
+ complex(0.0, 8.0),
+ complex(0.0, -6.0),
+ ],
+ centric_flags=[True, True, False, False],
+ )
+ return _FakeMillerArray(
+ hkls_4,
+ [
+ complex(2.0, 0.0),
+ complex(-1.5, 0.0),
+ complex(0.0, 1.25),
+ complex(0.0, -0.75),
+ ],
+ centric_flags=[True, True, False, False],
+ )
+
+ def _fmodel_manager(
+ xray_structure: Any,
+ r_free_flags: Any,
+ f_obs: Any,
+ twin_law: Any = None,
+ ) -> Any:
+ if fail_in_fmodel:
+ raise RuntimeError("Simulated f_model outlier failure")
+
+ def _update_all_scales(
+ update_f_part1: bool = True,
+ log: Any = None,
+ ) -> None:
+ if log is not None:
+ log.write("fmodel.update_all_scales log\n")
+ scales_completed[0] = True
+
+ def _f_obs_free() -> _FakeMillerArray:
+ if fail_in_free_array:
+ raise RuntimeError(
+ "Injected synthetic f_obs_free failure"
+ )
+ return _FakeMillerArray(hkls_t, f_obs_t, sigf_t)
+
+ effective_r_work = (
+ native_r_work_override
+ if native_r_work_override is not None
+ else (20.0 / 300.0)
+ )
+ return types.SimpleNamespace(
+ xray_structure=_ScopedXrayStructure(),
+ update_all_scales=_update_all_scales,
+ map_coefficients=_map_coefficients,
+ f_obs=lambda: _FakeMillerArray(
+ hkls_4,
+ [0.0, *f_obs_4[1:]]
+ if nonpositive_final_obs
+ else f_obs_4,
+ sigf_4,
+ ),
+ r_free_flags=lambda: _FakeMillerArray(hkls_4, flags_4),
+ f_obs_work=lambda: _FakeMillerArray(
+ hkls_w,
+ [0.0, *f_obs_w[1:]]
+ if nonpositive_final_obs
+ else f_obs_w,
+ sigf_w,
+ ),
+ f_obs_free=_f_obs_free,
+ f_model_work=lambda: _FakeMillerArray(
+ hkls_w,
+ [complex(float("nan"), 144.0), *fm_w[1:]]
+ if nonfinite_work_model_coeff
+ else fm_w,
+ ),
+ f_model_free=lambda: _FakeMillerArray(hkls_t, fm_t),
+ f_model_scaled_with_k1_w=lambda: _FakeMillerArray(
+ hkls_w,
+ [complex(float("nan"), 72.0), *fm_scaled_w[1:]]
+ if nonfinite_work_model_coeff
+ else fm_scaled_w,
+ ),
+ f_model_scaled_with_k1_t=lambda: _FakeMillerArray(
+ hkls_t, fm_scaled_t
+ ),
+ r_work=lambda: effective_r_work,
+ r_free=lambda: 50.0 / 200.0,
+ r_all=lambda: 70.6 / 500.0,
+ scale_k1_w=lambda: 0.5,
+ scale_k1_t=lambda: 0.6,
+ scale_k1=lambda: 0.54,
+ )
+
+ mmtbx_fmodel.manager = _fmodel_manager
+ mmtbx_fmodel.f_model = mmtbx_fmodel_mod
+ mmtbx_fmodel_mod.manager = _fmodel_manager
+
+ iotbx_mod.pdb = iotbx_pdb
+ iotbx_mod.mtz = iotbx_mtz
+ iotbx_mod.reflection_file_reader = iotbx_rfr
+ iotbx_mod.reflection_file_utils = iotbx_rfu
+ iotbx_mod.extract_xtal_data = iotbx_exd
+ mmtbx_mod.f_model = mmtbx_fmodel
+ mmtbx_mod.map_tools = mmtbx_map_tools
+ mmtbx_mod.model = mmtbx_model
+ mmtbx_mod.model_vs_data = mmtbx_mvd
+ cctbx_mod.crystal = cctbx_crystal
+
+ return {
+ "iotbx": iotbx_mod,
+ "iotbx.pdb": iotbx_pdb,
+ "iotbx.mtz": iotbx_mtz,
+ "iotbx.reflection_file_reader": iotbx_rfr,
+ "iotbx.reflection_file_utils": iotbx_rfu,
+ "iotbx.extract_xtal_data": iotbx_exd,
+ "mmtbx": mmtbx_mod,
+ "mmtbx.f_model": mmtbx_fmodel,
+ "mmtbx.f_model.f_model": mmtbx_fmodel_mod,
+ "mmtbx.map_tools": mmtbx_map_tools,
+ "mmtbx.model": mmtbx_model,
+ "mmtbx.model_vs_data": mmtbx_mvd,
+ "cctbx": cctbx_mod,
+ "cctbx.crystal": cctbx_crystal,
+ }
+
+ with tempfile.TemporaryDirectory() as tmp:
+ tmp_dir = Path(tmp).resolve(strict=True)
+ fake_prefix = tmp_dir / "opt_phenix"
+ fake_prefix.mkdir(parents=True, exist_ok=True)
+ resolved_fake_prefix = str(fake_prefix.resolve(strict=True))
+ pdb_path = tmp_dir / "model.pdb"
+ mtz_path = tmp_dir / "obs.mtz"
+ params_path = tmp_dir / "params.json"
+ out_json = tmp_dir / "rfactors.json"
+ out_maps = tmp_dir / "phx_maps.mtz"
+ out_pair = tmp_dir / "PHX_OBS_PAIR_ONLY.mtz"
+ out_diag = tmp_dir / "diagnostics.json"
+
+ pdb_path.write_text("HEADER TEST\nEND\n", encoding="utf-8")
+ ledger, fixture_payload, vendor_shas, _, _ = (
+ _make_reconciliation_fixture()
+ )
+ mtz_path.write_bytes(b"MTZ_DUMMY")
+ params = dict(onesar_phenix_measure.EXPECTED_PARAMETERS)
+ params["subject_id"] = "R4"
+ params["expected_resolved_prefix"] = resolved_fake_prefix
+ params["expected_phenix_version"] = toolchain.PHENIX_VERSION
+ params["map_coefficients"] = dict(
+ onesar_phenix_measure.EXPECTED_MAP_PARAMETERS
+ )
+ params_path.write_text(
+ json.dumps(params, indent=2) + "\n", encoding="utf-8"
+ )
+
+ fake_runtime_env = copy.deepcopy(
+ fixture_payload["runtime_environment"]
+ )
+ fake_runtime_env["sys_prefix"] = resolved_fake_prefix
+ fake_runtime_env["resolved_sys_prefix"] = resolved_fake_prefix
+ fake_runtime_env["sys_executable"] = (
+ f"{resolved_fake_prefix}/bin/python3.9"
+ )
+ fake_runtime_env["resolved_sys_executable"] = (
+ f"{resolved_fake_prefix}/bin/python3.9"
+ )
+ for mod_key, rel in onesar_rerun.PHENIX_VENDOR_SOURCE_RELPATHS.items():
+ mod_p = f"{resolved_fake_prefix}/lib/python3.9/site-packages/{rel}"
+ fake_runtime_env["vendor_modules"][mod_key]["configured_path"] = (
+ mod_p
+ )
+ fake_runtime_env["vendor_modules"][mod_key]["resolved_path"] = (
+ mod_p
+ )
+
+ # 1. Success path in-process:
+ with (
+ mock.patch.dict(sys.modules, _make_vendor_modules()),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ result_payload = (
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json,
+ out_maps_mtz_path=out_maps,
+ out_obs_pair_mtz_path=out_pair,
+ out_diagnostics_json_path=out_diag,
+ )
+ )
+
+ self.assertTrue(out_json.is_file())
+ self.assertTrue(out_maps.is_file())
+ self.assertTrue(out_pair.is_file())
+ self.assertTrue(out_diag.is_file())
+ self.assertEqual(
+ result_payload["diagnostics"]["current_phase"], "completed"
+ )
+ self.assertIn(
+ "update_all_scales",
+ result_payload["diagnostics"]["phase_history"],
+ )
+ reconciled = onesar_rerun.reconcile_phenix_rfactor_output(
+ result_payload,
+ ledger,
+ expected_vendor_source_sha256=vendor_shas,
+ expected_resolved_prefix=resolved_fake_prefix,
+ phx_maps_mtz_path=out_maps,
+ phx_obs_pair_only_mtz_path=out_pair,
+ phenix_diagnostics_path=out_diag,
+ )
+ self.assertEqual(reconciled["reconciliation_status"], "verified")
+
+ # 1b. Direct _PersistentTeeLogStream live-flush verification before finalize():
+ live_diag_path = tmp_dir / "live_tee_diag.json"
+ live_log_path = tmp_dir / "live_tee.log"
+ live_diag: dict[str, Any] = {
+ "status": "running",
+ "current_phase": "update_all_scales",
+ "fmodel_update_all_scales_log": "",
+ }
+ tee_stream = onesar_phenix_measure._PersistentTeeLogStream(
+ "fmodel.update_all_scales",
+ live_diag,
+ "fmodel_update_all_scales_log",
+ diagnostics_json_path=live_diag_path,
+ log_file_path=live_log_path,
+ )
+ tee_stream.write("chunk_one\n")
+ # Persisted on disk immediately on write() without calling finalize():
+ self.assertEqual(
+ live_log_path.read_text(encoding="utf-8"), "chunk_one\n"
+ )
+ self.assertEqual(
+ json.loads(live_diag_path.read_text(encoding="utf-8"))[
+ "fmodel_update_all_scales_log"
+ ],
+ "chunk_one\n",
+ )
+ tee_stream.write("chunk_two")
+ self.assertEqual(
+ live_log_path.read_text(encoding="utf-8"),
+ "chunk_one\nchunk_two",
+ )
+ self.assertEqual(
+ json.loads(live_diag_path.read_text(encoding="utf-8"))[
+ "fmodel_update_all_scales_log"
+ ],
+ "chunk_one\nchunk_two",
+ )
+
+ # 1c. Deterministic interrupted-checkpoint test (#1014 Follow-up 2, #1017):
+ # A failed/interrupted checkpoint write or promotion must preserve the prior
+ # complete JSON checkpoint on disk and unlink the temporary sibling.
+ with self.subTest(case="1c_interrupted_checkpoint"):
+ self._verify_atomic_checkpoint_interrupted_write_and_promotion(
+ tmp_dir
+ )
+
+ # 2. Failure in extract_xtal_data retains df.log in diagnostics file (#1014, #1017):
+ with self.subTest(case="2_extract_xtal_data_err"):
+ out_diag_err = tmp_dir / "diagnostics_exd_err.json"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(
+ df_err=["Bad R-free flags column"]
+ ),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ RuntimeError, "extract_xtal_data reported errors"
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=tmp_dir / "unused1.json",
+ out_maps_mtz_path=tmp_dir / "unused1.mtz",
+ out_obs_pair_mtz_path=tmp_dir / "unused1_pair.mtz",
+ out_diagnostics_json_path=out_diag_err,
+ )
+ self.assertTrue(out_diag_err.is_file())
+ diag_err_doc = json.loads(
+ out_diag_err.read_text(encoding="utf-8")
+ )
+ self.assertEqual(diag_err_doc["status"], "failed")
+ self.assertEqual(
+ diag_err_doc["current_phase"], "extract_xtal_data"
+ )
+ self.assertEqual(
+ diag_err_doc["failure"]["failed_phase"],
+ "extract_xtal_data",
+ )
+ self.assertIn(
+ "Detailed extract_xtal_data log output",
+ diag_err_doc["extract_xtal_data_log"],
+ )
+
+ # 2b. Exception inside extract_xtal_data.run before returning records
+ # constructor_log_limitation for the unreturned internal StringIO (#1014):
+ with self.subTest(case="2b_extract_xtal_data_raise"):
+ out_diag_exd_raise = tmp_dir / "diagnostics_exd_raise.json"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(raise_in_extract=True),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ RuntimeError,
+ "extract_xtal_data constructor raised before returning",
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=tmp_dir / "unused1b.json",
+ out_maps_mtz_path=tmp_dir / "unused1b.mtz",
+ out_obs_pair_mtz_path=tmp_dir / "unused1b_pair.mtz",
+ out_diagnostics_json_path=out_diag_exd_raise,
+ )
+ self.assertTrue(out_diag_exd_raise.is_file())
+ diag_exd_raise_doc = json.loads(
+ out_diag_exd_raise.read_text(encoding="utf-8")
+ )
+ self.assertEqual(diag_exd_raise_doc["status"], "failed")
+ self.assertEqual(
+ diag_exd_raise_doc["current_phase"], "extract_xtal_data"
+ )
+ self.assertIsNone(diag_exd_raise_doc["extract_xtal_data_log"])
+ self.assertIn(
+ "internal self.log StringIO on the unreturned instance is not accessible",
+ diag_exd_raise_doc["failure"]["constructor_log_limitation"],
+ )
+
+ # 3. Failure in mmtbx.f_model.manager records constructor_log_limitation (#1014):
+ with self.subTest(case="3_fmodel_manager_init_fail"):
+ out_diag_fm = tmp_dir / "diagnostics_fmodel_err.json"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(fail_in_fmodel=True),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ RuntimeError, "Simulated f_model outlier failure"
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=tmp_dir / "unused2.json",
+ out_maps_mtz_path=tmp_dir / "unused2.mtz",
+ out_obs_pair_mtz_path=tmp_dir / "unused2_pair.mtz",
+ out_diagnostics_json_path=out_diag_fm,
+ )
+ self.assertTrue(out_diag_fm.is_file())
+ diag_fm_doc = json.loads(
+ out_diag_fm.read_text(encoding="utf-8")
+ )
+ self.assertEqual(diag_fm_doc["status"], "failed")
+ self.assertEqual(
+ diag_fm_doc["failure"]["failed_phase"],
+ "fmodel_manager_init",
+ )
+ self.assertIn(
+ "accepts no log parameter",
+ diag_fm_doc["failure"]["constructor_log_limitation"],
+ )
+ self.assertIn(
+ "model.manager initialization log",
+ diag_fm_doc["model_manager_log"],
+ )
+ partial_fm_doc = json.loads(
+ (tmp_dir / "unused2_partial_evidence.json").read_text(
+ encoding="utf-8"
+ )
+ )
+ self.assertEqual(
+ partial_fm_doc["xray_structure_snapshots_raw"]["before"],
+ result_payload["xray_structure_invariance"]["snapshots"][
+ "before"
+ ],
+ )
+ self.assertIsNone(
+ partial_fm_doc["xray_structure_snapshots_raw"][
+ "after_model"
+ ]
+ )
+ self.assertIsNone(
+ partial_fm_doc["xray_structure_snapshots_raw"][
+ "after_fmodel"
+ ]
+ )
+ self.assertFalse(
+ partial_fm_doc["xray_structure_snapshots_raw"][
+ "invariance_verified"
+ ]
+ )
+ self.assertIsNone(partial_fm_doc["xray_structure_invariance"])
+
+ # 3a1. Scatterer count change during scaling retains all three raw scatterer
+ # snapshots (counts 1, 2, 2) while leaving verified invariance None (#1014, Task 120):
+ with self.subTest(case="3a1_scatterer_count_change"):
+ out_json_scc = tmp_dir / "rfactors_scc.json"
+ out_partial_scc = (
+ tmp_dir / "rfactors_scc_partial_evidence.json"
+ )
+ out_diag_scc = tmp_dir / "diagnostics_scc.json"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(
+ mutate_scatterer_count_in_scales=True
+ ),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ ValueError,
+ "Scatterer count changed across f_model scaling",
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_scc,
+ out_maps_mtz_path=tmp_dir / "scc_maps.mtz",
+ out_obs_pair_mtz_path=tmp_dir / "scc_pair.mtz",
+ out_diagnostics_json_path=out_diag_scc,
+ )
+ self.assertFalse(out_json_scc.is_file())
+ self.assertTrue(out_partial_scc.is_file())
+ partial_scc_doc = json.loads(
+ out_partial_scc.read_text(encoding="utf-8")
+ )
+ self.assertEqual(
+ partial_scc_doc["status"], "failed_partial_unverified"
+ )
+ self.assertFalse(partial_scc_doc["reusable_cache_output"])
+ self.assertFalse(partial_scc_doc["canonical_eval_qds_eligible"])
+ self.assertFalse(partial_scc_doc["rfactor_replay_verified"])
+ self.assertFalse(partial_scc_doc["map_export_verified"])
+ self.assertIsNone(partial_scc_doc["xray_structure_invariance"])
+ raw_snaps_scc = partial_scc_doc["xray_structure_snapshots_raw"]
+ self.assertFalse(raw_snaps_scc["invariance_verified"])
+ self.assertEqual(
+ raw_snaps_scc["before"],
+ result_payload["xray_structure_invariance"]["snapshots"][
+ "before"
+ ],
+ )
+ self.assertEqual(raw_snaps_scc["before"]["n_scatterers"], 1)
+ self.assertEqual(
+ raw_snaps_scc["after_model"]["n_scatterers"], 2
+ )
+ self.assertEqual(
+ len(raw_snaps_scc["after_model"]["records"]), 2
+ )
+ self.assertEqual(
+ raw_snaps_scc["after_fmodel"]["n_scatterers"], 2
+ )
+ self.assertEqual(
+ len(raw_snaps_scc["after_fmodel"]["records"]), 2
+ )
+ self.assertEqual(
+ partial_scc_doc["completed_stages"],
+ [
+ "extract_xtal_data",
+ "xray_structure_snapshot_before",
+ "xray_structure_snapshot_after_model",
+ "xray_structure_snapshot_after_fmodel",
+ ],
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "Partial/unverified PHENIX evidence artifact cannot be reconciled",
+ ):
+ onesar_rerun.reconcile_phenix_rfactor_output(
+ partial_scc_doc,
+ ledger,
+ expected_vendor_source_sha256=vendor_shas,
+ expected_resolved_prefix=resolved_fake_prefix,
+ )
+
+ # 3a2. Scatterer occupancy change during scaling retains structured raw snapshots
+ # in xray_structure_snapshots_raw while leaving verified invariance None (#1014, Task 120):
+ with self.subTest(case="3a2_scatterer_occupancy_change"):
+ out_json_soc = tmp_dir / "rfactors_soc.json"
+ out_partial_soc = (
+ tmp_dir / "rfactors_soc_partial_evidence.json"
+ )
+ out_diag_soc = tmp_dir / "diagnostics_soc.json"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(
+ mutate_scatterer_occupancy_in_scales=True
+ ),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ ValueError,
+ "Scatterer snapshot mutated during scaling",
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_soc,
+ out_maps_mtz_path=tmp_dir / "soc_maps.mtz",
+ out_obs_pair_mtz_path=tmp_dir / "soc_pair.mtz",
+ out_diagnostics_json_path=out_diag_soc,
+ )
+ self.assertFalse(out_json_soc.is_file())
+ self.assertTrue(out_partial_soc.is_file())
+ partial_soc_doc = json.loads(
+ out_partial_soc.read_text(encoding="utf-8")
+ )
+ self.assertIsNone(partial_soc_doc["xray_structure_invariance"])
+ raw_snaps_soc = partial_soc_doc["xray_structure_snapshots_raw"]
+ self.assertFalse(raw_snaps_soc["invariance_verified"])
+ self.assertEqual(
+ raw_snaps_soc["before"]["records"][0]["occupancy"], 1.0
+ )
+ self.assertEqual(
+ raw_snaps_soc["after_model"]["records"][0]["occupancy"],
+ 0.5,
+ )
+ self.assertEqual(
+ raw_snaps_soc["after_fmodel"]["records"][0]["occupancy"],
+ 0.5,
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "Partial/unverified PHENIX evidence artifact cannot be reconciled",
+ ):
+ onesar_rerun.reconcile_phenix_rfactor_output(
+ partial_soc_doc,
+ ledger,
+ expected_vendor_source_sha256=vendor_shas,
+ expected_resolved_prefix=resolved_fake_prefix,
+ )
+
+ # 3a2b. Non-finite post-scaling occupancy retains all captured scatterer records
+ # and exact IEEE-754 bits in xray_structure_snapshots_raw["after_model"] before
+ # snapshot_scatterer_records raises ValueError (#1014, Task 128):
+ with self.subTest(
+ case="3a2b_scatterer_nonfinite_occupancy_after_scales"
+ ):
+ out_json_snfo = tmp_dir / "rfactors_snfo.json"
+ out_partial_snfo = (
+ tmp_dir / "rfactors_snfo_partial_evidence.json"
+ )
+ out_diag_snfo = tmp_dir / "diagnostics_snfo.json"
+ snfo_nan_val = struct.unpack(
+ ">d", struct.pack(">Q", 0x7FF8000000000123)
+ )[0]
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(
+ nonfinite_scatterer_occupancy_in_scales=snfo_nan_val
+ ),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ ValueError,
+ "Invalid scatterer record at index 0",
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_snfo,
+ out_maps_mtz_path=tmp_dir / "snfo_maps.mtz",
+ out_obs_pair_mtz_path=tmp_dir / "snfo_pair.mtz",
+ out_diagnostics_json_path=out_diag_snfo,
+ )
+ self.assertFalse(out_json_snfo.is_file())
+ self.assertTrue(out_partial_snfo.is_file())
+ partial_snfo_doc = json.loads(
+ out_partial_snfo.read_text(encoding="utf-8"),
+ parse_constant=lambda tok: (_ for _ in ()).throw(
+ ValueError(f"Forbidden non-strict JSON constant: {tok}")
+ ),
+ )
+ self.assertEqual(
+ partial_snfo_doc["status"], "failed_partial_unverified"
+ )
+ self.assertIsNone(partial_snfo_doc["xray_structure_invariance"])
+ raw_snaps_snfo = partial_snfo_doc[
+ "xray_structure_snapshots_raw"
+ ]
+ self.assertFalse(raw_snaps_snfo["invariance_verified"])
+ self.assertEqual(
+ raw_snaps_snfo["before"]["n_scatterers"], 1
+ )
+ self.assertIsNotNone(raw_snaps_snfo["before"]["sha256"])
+ after_model_snfo = raw_snaps_snfo["after_model"]
+ self.assertIsNotNone(after_model_snfo)
+ self.assertEqual(after_model_snfo["n_scatterers"], 1)
+ self.assertIsNone(after_model_snfo["sha256"])
+ self.assertEqual(len(after_model_snfo["records"]), 1)
+ snfo_rec0 = after_model_snfo["records"][0]
+ self.assertEqual(snfo_rec0["label"], 'PDB=" N VAL A 1 "')
+ self.assertEqual(snfo_rec0["scattering_type"], "N")
+ self.assertEqual(snfo_rec0["site_frac"], [0.1, 0.2, 0.3])
+ self.assertEqual(
+ snfo_rec0["site_cart"], [6.4897, 15.6646, 11.6376]
+ )
+ self.assertEqual(snfo_rec0["u_iso"], 0.25)
+ occ_tag = snfo_rec0["occupancy"]
+ self.assertIsInstance(occ_tag, dict)
+ self.assertTrue(occ_tag["__nonfinite_ieee754_float64__"])
+ self.assertEqual(occ_tag["kind"], "nan")
+ self.assertEqual(
+ occ_tag["ieee754_binary64_hex"], "0x7ff8000000000123"
+ )
+ self.assertEqual(occ_tag["sign_bit"], 0)
+ self.assertEqual(
+ occ_tag["field_path"],
+ "xray_structure_snapshots_raw.after_model.records[0].occupancy",
+ )
+ self.assertIsNone(raw_snaps_snfo["after_fmodel"])
+ self.assertEqual(
+ partial_snfo_doc["completed_stages"],
+ ["extract_xtal_data", "xray_structure_snapshot_before"],
+ )
+ self.assertEqual(
+ [
+ item["field_path"]
+ for item in partial_snfo_doc["nonfinite_raw_values"]
+ ],
+ [
+ "xray_structure_snapshots_raw.after_model.records[0].occupancy"
+ ],
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "Partial/unverified PHENIX evidence artifact cannot be reconciled",
+ ):
+ onesar_rerun.reconcile_phenix_rfactor_output(
+ partial_snfo_doc,
+ ledger,
+ expected_vendor_source_sha256=vendor_shas,
+ expected_resolved_prefix=resolved_fake_prefix,
+ )
+
+ # 3a2c. Non-finite post-scaling isotropic ADP (u_iso) retains all captured scatterer
+ # records and exact IEEE-754 bits in xray_structure_snapshots_raw["after_model"] (#1014, Task 128):
+ with self.subTest(case="3a2c_scatterer_nonfinite_adp_after_scales"):
+ out_json_snfu = tmp_dir / "rfactors_snfu.json"
+ out_partial_snfu = (
+ tmp_dir / "rfactors_snfu_partial_evidence.json"
+ )
+ out_diag_snfu = tmp_dir / "diagnostics_snfu.json"
+ snfu_nan_val = struct.unpack(
+ ">d", struct.pack(">Q", 0xFFF8000000000456)
+ )[0]
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(
+ nonfinite_scatterer_u_iso_in_scales=snfu_nan_val
+ ),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ ValueError,
+ "Invalid scatterer record at index 0",
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_snfu,
+ out_maps_mtz_path=tmp_dir / "snfu_maps.mtz",
+ out_obs_pair_mtz_path=tmp_dir / "snfu_pair.mtz",
+ out_diagnostics_json_path=out_diag_snfu,
+ )
+ self.assertFalse(out_json_snfu.is_file())
+ self.assertTrue(out_partial_snfu.is_file())
+ partial_snfu_doc = json.loads(
+ out_partial_snfu.read_text(encoding="utf-8"),
+ parse_constant=lambda tok: (_ for _ in ()).throw(
+ ValueError(f"Forbidden non-strict JSON constant: {tok}")
+ ),
+ )
+ self.assertIsNone(partial_snfu_doc["xray_structure_invariance"])
+ raw_snaps_snfu = partial_snfu_doc[
+ "xray_structure_snapshots_raw"
+ ]
+ self.assertFalse(raw_snaps_snfu["invariance_verified"])
+ after_model_snfu = raw_snaps_snfu["after_model"]
+ self.assertIsNotNone(after_model_snfu)
+ self.assertEqual(after_model_snfu["n_scatterers"], 1)
+ self.assertIsNone(after_model_snfu["sha256"])
+ self.assertEqual(
+ after_model_snfu["records"][0]["occupancy"], 1.0
+ )
+ uiso_tag = after_model_snfu["records"][0]["u_iso"]
+ self.assertIsInstance(uiso_tag, dict)
+ self.assertTrue(uiso_tag["__nonfinite_ieee754_float64__"])
+ self.assertEqual(uiso_tag["kind"], "nan")
+ self.assertEqual(
+ uiso_tag["ieee754_binary64_hex"], "0xfff8000000000456"
+ )
+ self.assertEqual(uiso_tag["sign_bit"], 1)
+ self.assertEqual(
+ uiso_tag["field_path"],
+ "xray_structure_snapshots_raw.after_model.records[0].u_iso",
+ )
+ self.assertIsNone(raw_snaps_snfu["after_fmodel"])
+ self.assertEqual(
+ partial_snfu_doc["completed_stages"],
+ ["extract_xtal_data", "xray_structure_snapshot_before"],
+ )
+ self.assertEqual(
+ [
+ item["field_path"]
+ for item in partial_snfu_doc["nonfinite_raw_values"]
+ ],
+ [
+ "xray_structure_snapshots_raw.after_model.records[0].u_iso"
+ ],
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "Partial/unverified PHENIX evidence artifact cannot be reconciled",
+ ):
+ onesar_rerun.reconcile_phenix_rfactor_output(
+ partial_snfu_doc,
+ ledger,
+ expected_vendor_source_sha256=vendor_shas,
+ expected_resolved_prefix=resolved_fake_prefix,
+ )
+
+ # 3a2d. Post-scaling zero scatterer population retains n_scatterers=0 and records=[]
+ # in xray_structure_snapshots_raw["after_model"] before failing closed (#1014, Task 128):
+ with self.subTest(
+ case="3a2d_scatterer_empty_population_after_scales"
+ ):
+ out_json_semp = tmp_dir / "rfactors_semp.json"
+ out_partial_semp = (
+ tmp_dir / "rfactors_semp_partial_evidence.json"
+ )
+ out_diag_semp = tmp_dir / "diagnostics_semp.json"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(empty_scatterers_in_scales=True),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ ValueError,
+ "Scatterer snapshot cannot be empty",
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_semp,
+ out_maps_mtz_path=tmp_dir / "semp_maps.mtz",
+ out_obs_pair_mtz_path=tmp_dir / "semp_pair.mtz",
+ out_diagnostics_json_path=out_diag_semp,
+ )
+ self.assertFalse(out_json_semp.is_file())
+ self.assertTrue(out_partial_semp.is_file())
+ partial_semp_doc = json.loads(
+ out_partial_semp.read_text(encoding="utf-8")
+ )
+ self.assertIsNone(partial_semp_doc["xray_structure_invariance"])
+ raw_snaps_semp = partial_semp_doc[
+ "xray_structure_snapshots_raw"
+ ]
+ self.assertFalse(raw_snaps_semp["invariance_verified"])
+ self.assertEqual(
+ raw_snaps_semp["before"]["n_scatterers"], 1
+ )
+ self.assertEqual(
+ raw_snaps_semp["after_model"],
+ {"n_scatterers": 0, "sha256": None, "records": []},
+ )
+ self.assertIsNone(raw_snaps_semp["after_fmodel"])
+ self.assertEqual(
+ partial_semp_doc["completed_stages"],
+ ["extract_xtal_data", "xray_structure_snapshot_before"],
+ )
+ self.assertEqual(partial_semp_doc["nonfinite_raw_values"], [])
+ with self.assertRaisesRegex(
+ ValueError,
+ "Partial/unverified PHENIX evidence artifact cannot be reconciled",
+ ):
+ onesar_rerun.reconcile_phenix_rfactor_output(
+ partial_semp_doc,
+ ledger,
+ expected_vendor_source_sha256=vendor_shas,
+ expected_resolved_prefix=resolved_fake_prefix,
+ )
+
+ # 3a2e. Worker fails closed when runtime crystal_symmetry is missing or has
+ # inconsistent space_group_number / symmetry operators (#1025, Task 128):
+ missing_sg_symm = types.SimpleNamespace(
+ unit_cell=lambda: types.SimpleNamespace(
+ parameters=lambda: _fake_f32_cell
+ ),
+ space_group_info=lambda: types.SimpleNamespace(
+ symbol_and_number=lambda: "P 21 21 21 (No. 19)",
+ ),
+ is_similar_symmetry=lambda other: True,
+ )
+ mismatch_num_symm = types.SimpleNamespace(
+ unit_cell=lambda: types.SimpleNamespace(
+ parameters=lambda: _fake_f32_cell
+ ),
+ space_group=lambda: types.SimpleNamespace(
+ type=lambda: types.SimpleNamespace(number=lambda: 19),
+ all_ops=lambda: list(_fake_rt_mx_ops),
+ ),
+ space_group_info=lambda: types.SimpleNamespace(
+ symbol_and_number=lambda: "P 21 21 21 (No. 19)",
+ type=lambda: types.SimpleNamespace(number=lambda: 18),
+ ),
+ is_similar_symmetry=lambda other: True,
+ )
+ empty_ops_symm = types.SimpleNamespace(
+ unit_cell=lambda: types.SimpleNamespace(
+ parameters=lambda: _fake_f32_cell
+ ),
+ space_group=lambda: types.SimpleNamespace(
+ type=lambda: types.SimpleNamespace(number=lambda: 19),
+ all_ops=lambda: [],
+ ),
+ space_group_info=lambda: types.SimpleNamespace(
+ symbol_and_number=lambda: "P 21 21 21 (No. 19)",
+ type=lambda: types.SimpleNamespace(number=lambda: 19),
+ ),
+ is_similar_symmetry=lambda other: True,
+ )
+ for idx_sym, (sym_case_name, bad_sym_obj, bad_sym_pat) in enumerate(
+ (
+ (
+ "missing_space_group",
+ missing_sg_symm,
+ r"is missing space_group\(\)",
+ ),
+ (
+ "mismatched_group_and_info_number",
+ mismatch_num_symm,
+ r"space_group_info\(\)\.type\(\)\.number\(\) 18 disagrees with space_group\(\)\.type\(\)\.number\(\) 19",
+ ),
+ (
+ "empty_all_ops",
+ empty_ops_symm,
+ r"space_group\(\)\.all_ops\(\) returned no symmetry operations",
+ ),
+ )
+ ):
+ with self.subTest(
+ case=f"3a2e_worker_symmetry_{sym_case_name}"
+ ):
+ out_json_sym = (
+ tmp_dir / f"rfactors_symm_bad_{idx_sym}.json"
+ )
+ out_partial_sym = (
+ tmp_dir
+ / f"rfactors_symm_bad_{idx_sym}_partial_evidence.json"
+ )
+ out_maps_sym = (
+ tmp_dir / f"symm_bad_{idx_sym}_maps.mtz"
+ )
+ out_pair_sym = (
+ tmp_dir / f"symm_bad_{idx_sym}_pair.mtz"
+ )
+ out_diag_sym = (
+ tmp_dir / f"diag_symm_bad_{idx_sym}.json"
+ )
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(symmetry_override=bad_sym_obj),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(RuntimeError, bad_sym_pat):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_sym,
+ out_maps_mtz_path=out_maps_sym,
+ out_obs_pair_mtz_path=out_pair_sym,
+ out_diagnostics_json_path=out_diag_sym,
+ )
+ self.assertFalse(out_json_sym.is_file())
+ self.assertFalse(out_maps_sym.is_file())
+ self.assertFalse(out_pair_sym.is_file())
+ self.assertTrue(out_partial_sym.is_file())
+ self.assertTrue(out_diag_sym.is_file())
+ partial_sym_doc = json.loads(
+ out_partial_sym.read_text(encoding="utf-8")
+ )
+ diag_sym_doc = json.loads(
+ out_diag_sym.read_text(encoding="utf-8")
+ )
+ self.assertEqual(
+ partial_sym_doc["status"], "failed_partial_unverified"
+ )
+ self.assertEqual(
+ partial_sym_doc["failure"]["failed_phase"], "pdb_input"
+ )
+ self.assertEqual(
+ partial_sym_doc["failure"]["exception_type"],
+ "RuntimeError",
+ )
+ self.assertRegex(
+ partial_sym_doc["failure"]["exception_message"],
+ bad_sym_pat,
+ )
+ self.assertEqual(diag_sym_doc["status"], "failed")
+ self.assertEqual(
+ diag_sym_doc["current_phase"], "pdb_input"
+ )
+ self.assertEqual(
+ diag_sym_doc["partial_evidence_status"],
+ "failed_partial_unverified",
+ )
+ self.assertEqual(
+ diag_sym_doc["failure"]["exception_type"],
+ "RuntimeError",
+ )
+ self.assertRegex(
+ diag_sym_doc["failure"]["exception_message"],
+ bad_sym_pat,
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "Partial/unverified PHENIX evidence artifact cannot be reconciled",
+ ):
+ onesar_rerun.reconcile_phenix_rfactor_output(
+ partial_sym_doc,
+ ledger,
+ expected_vendor_source_sha256=vendor_shas,
+ expected_resolved_prefix=resolved_fake_prefix,
+ )
+
+ # 3a3. Non-finite model coefficient fails in compute_rfactor_from_arrays after
+ # checkpointing final_effective, actual_work, and actual_free with strict-JSON
+ # IEEE-754 nonfinite tags (#1014, Task 120):
+ with self.subTest(case="3a3_nonfinite_work_model_coefficient"):
+ out_json_nf = tmp_dir / "rfactors_nonfinite.json"
+ out_partial_nf = (
+ tmp_dir / "rfactors_nonfinite_partial_evidence.json"
+ )
+ out_diag_nf = tmp_dir / "diagnostics_nonfinite.json"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(nonfinite_work_model_coeff=True),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ ValueError,
+ r"Non-finite amplitude at index 0: fo=100\.0, fm=nan",
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_nf,
+ out_maps_mtz_path=tmp_dir / "nf_maps.mtz",
+ out_obs_pair_mtz_path=tmp_dir / "nf_pair.mtz",
+ out_diagnostics_json_path=out_diag_nf,
+ )
+ self.assertFalse(out_json_nf.is_file())
+ self.assertTrue(out_partial_nf.is_file())
+ # Verify strict RFC 8259 JSON compliance (no bare NaN/Infinity tokens):
+ partial_nf_doc = json.loads(
+ out_partial_nf.read_text(encoding="utf-8"),
+ parse_constant=lambda tok: (_ for _ in ()).throw(
+ ValueError(f"Forbidden non-strict JSON constant: {tok}")
+ ),
+ )
+ self.assertEqual(
+ partial_nf_doc["status"], "failed_partial_unverified"
+ )
+ self.assertFalse(partial_nf_doc["rfactor_replay_verified"])
+ self.assertIsNone(partial_nf_doc["scales_and_rfactors"])
+ self.assertIsNone(partial_nf_doc["rfactor_replay_comparison"])
+ self.assertEqual(
+ partial_nf_doc["extracted_pre_fmodel"],
+ result_payload["extracted_pre_fmodel"],
+ )
+ self.assertEqual(
+ partial_nf_doc["final_effective"],
+ result_payload["final_effective"],
+ )
+ self.assertEqual(
+ partial_nf_doc["actual_free"],
+ result_payload["actual_free"],
+ )
+ self.assertEqual(len(partial_nf_doc["actual_work"]), 2)
+ self.assertEqual(
+ partial_nf_doc["actual_work"][1],
+ result_payload["actual_work"][1],
+ )
+ nf_row0 = partial_nf_doc["actual_work"][0]
+ self.assertEqual(nf_row0["hkl"], [1, 0, 0])
+ self.assertEqual(nf_row0["f_obs"], 100.0)
+ self.assertEqual(nf_row0["sigf_obs"], 2.0)
+ self.assertEqual(nf_row0["f_model_imag"], 144.0)
+ self.assertEqual(
+ nf_row0["f_model_scaled_with_k1_w_imag"], 72.0
+ )
+ expected_nf_fields = [
+ "f_model_real",
+ "f_model_amplitude",
+ "f_model_scaled_with_k1_w_real",
+ "f_model_scaled_with_k1_w_amplitude",
+ ]
+ for nf_field in expected_nf_fields:
+ tag_obj = nf_row0[nf_field]
+ self.assertIsInstance(tag_obj, dict)
+ self.assertTrue(tag_obj["__nonfinite_ieee754_float64__"])
+ self.assertEqual(tag_obj["kind"], "nan")
+ self.assertEqual(tag_obj["repr"], "nan")
+ self.assertEqual(
+ tag_obj["field_path"], f"actual_work[0].{nf_field}"
+ )
+ self.assertTrue(
+ tag_obj["ieee754_binary64_hex"].startswith("0x")
+ )
+ self.assertEqual(
+ [
+ item["field_path"]
+ for item in partial_nf_doc["nonfinite_raw_values"]
+ ],
+ [f"actual_work[0].{f}" for f in expected_nf_fields],
+ )
+ self.assertIn(
+ "final_effective_raw", partial_nf_doc["completed_stages"]
+ )
+ self.assertIn(
+ "actual_work_raw", partial_nf_doc["completed_stages"]
+ )
+ self.assertIn(
+ "actual_free_raw", partial_nf_doc["completed_stages"]
+ )
+ self.assertNotIn(
+ "rfactor_raw_arrays_and_scalars",
+ partial_nf_doc["completed_stages"],
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "Partial/unverified PHENIX evidence artifact cannot be reconciled",
+ ):
+ onesar_rerun.reconcile_phenix_rfactor_output(
+ partial_nf_doc,
+ ledger,
+ expected_vendor_source_sha256=vendor_shas,
+ expected_resolved_prefix=resolved_fake_prefix,
+ )
+
+ # 3a4. Non-positive observation (fo=0.0) fails in compute_rfactor_from_arrays
+ # while retaining completed final_effective, actual_work, and actual_free (#1014, Task 120):
+ with self.subTest(case="3a4_nonpositive_final_observation"):
+ out_json_np = tmp_dir / "rfactors_nonpos.json"
+ out_partial_np = (
+ tmp_dir / "rfactors_nonpos_partial_evidence.json"
+ )
+ out_diag_np = tmp_dir / "diagnostics_nonpos.json"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(nonpositive_final_obs=True),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ ValueError,
+ r"Invalid amplitude sign at index 0: fo=0\.0, fm=90\.0",
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_np,
+ out_maps_mtz_path=tmp_dir / "np_maps.mtz",
+ out_obs_pair_mtz_path=tmp_dir / "np_pair.mtz",
+ out_diagnostics_json_path=out_diag_np,
+ )
+ self.assertFalse(out_json_np.is_file())
+ self.assertTrue(out_partial_np.is_file())
+ partial_np_doc = json.loads(
+ out_partial_np.read_text(encoding="utf-8")
+ )
+ self.assertFalse(partial_np_doc["rfactor_replay_verified"])
+ self.assertIsNone(partial_np_doc["scales_and_rfactors"])
+ self.assertIsNone(partial_np_doc["rfactor_replay_comparison"])
+ self.assertEqual(partial_np_doc["nonfinite_raw_values"], [])
+ self.assertEqual(len(partial_np_doc["final_effective"]), 4)
+ self.assertEqual(
+ partial_np_doc["final_effective"][0]["f_obs"], 0.0
+ )
+ self.assertEqual(
+ partial_np_doc["final_effective"][1:],
+ result_payload["final_effective"][1:],
+ )
+ self.assertEqual(len(partial_np_doc["actual_work"]), 2)
+ self.assertEqual(
+ partial_np_doc["actual_work"][0]["f_obs"], 0.0
+ )
+ self.assertEqual(
+ partial_np_doc["actual_work"][1],
+ result_payload["actual_work"][1],
+ )
+ self.assertEqual(
+ partial_np_doc["actual_free"],
+ result_payload["actual_free"],
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "Partial/unverified PHENIX evidence artifact cannot be reconciled",
+ ):
+ onesar_rerun.reconcile_phenix_rfactor_output(
+ partial_np_doc,
+ ledger,
+ expected_vendor_source_sha256=vendor_shas,
+ expected_resolved_prefix=resolved_fake_prefix,
+ )
+
+ # 3a5. Failure while constructing actual_free preserves already completed
+ # final_effective and actual_work arrays (#1014, Task 120):
+ with self.subTest(
+ case="3a5_later_free_array_failure_preserves_final_and_work"
+ ):
+ out_json_ffail = tmp_dir / "rfactors_freefail.json"
+ out_partial_ffail = (
+ tmp_dir / "rfactors_freefail_partial_evidence.json"
+ )
+ out_diag_ffail = tmp_dir / "diagnostics_freefail.json"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(fail_in_free_array=True),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ RuntimeError, "Injected synthetic f_obs_free failure"
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_ffail,
+ out_maps_mtz_path=tmp_dir / "ffail_maps.mtz",
+ out_obs_pair_mtz_path=tmp_dir / "ffail_pair.mtz",
+ out_diagnostics_json_path=out_diag_ffail,
+ )
+ self.assertFalse(out_json_ffail.is_file())
+ self.assertTrue(out_partial_ffail.is_file())
+ partial_ffail_doc = json.loads(
+ out_partial_ffail.read_text(encoding="utf-8")
+ )
+ self.assertEqual(
+ partial_ffail_doc["final_effective"],
+ result_payload["final_effective"],
+ )
+ self.assertEqual(
+ partial_ffail_doc["actual_work"],
+ result_payload["actual_work"],
+ )
+ self.assertIsNone(partial_ffail_doc["actual_free"])
+
+ # 3b. Deliberate native r_work mismatch fails before success output is written,
+ # while partial_evidence JSON on disk retains extracted/final/work/free HKLs,
+ # observations, complex model coefficients, scatterers, and both native and
+ # replayed scalar R values (#1014 Finding 4, #1017):
+ with self.subTest(case="3b_native_r_work_mismatch"):
+ out_json_rmis = tmp_dir / "rfactors_rmismatch.json"
+ out_partial_rmis = (
+ tmp_dir / "rfactors_rmismatch_partial_evidence.json"
+ )
+ out_diag_rmis = tmp_dir / "diagnostics_rmismatch.json"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(
+ native_r_work_override=(20.0 / 300.0 + 0.01)
+ ),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ RuntimeError,
+ r"Native fmodel\.r_work\(\) .* disagrees with array replay",
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_rmis,
+ out_maps_mtz_path=tmp_dir / "rmismatch_maps.mtz",
+ out_obs_pair_mtz_path=tmp_dir
+ / "rmismatch_pair.mtz",
+ out_diagnostics_json_path=out_diag_rmis,
+ )
+ self.assertFalse(out_json_rmis.is_file())
+ self.assertTrue(out_partial_rmis.is_file())
+ partial_rmis_doc = json.loads(
+ out_partial_rmis.read_text(encoding="utf-8")
+ )
+ self.assertEqual(
+ partial_rmis_doc["artifact_role"],
+ "partial_unverified_phenix_evidence",
+ )
+ self.assertFalse(partial_rmis_doc["reusable_cache_output"])
+ self.assertFalse(
+ partial_rmis_doc["canonical_eval_qds_eligible"]
+ )
+ self.assertEqual(
+ partial_rmis_doc["status"], "failed_partial_unverified"
+ )
+ self.assertFalse(partial_rmis_doc["rfactor_replay_verified"])
+ self.assertEqual(
+ partial_rmis_doc["extracted_pre_fmodel"],
+ result_payload["extracted_pre_fmodel"],
+ )
+ self.assertEqual(
+ partial_rmis_doc["final_effective"],
+ result_payload["final_effective"],
+ )
+ self.assertEqual(
+ partial_rmis_doc["actual_work"],
+ result_payload["actual_work"],
+ )
+ self.assertEqual(
+ partial_rmis_doc["actual_free"],
+ result_payload["actual_free"],
+ )
+ self.assertEqual(
+ partial_rmis_doc["xray_structure_invariance"],
+ result_payload["xray_structure_invariance"],
+ )
+ self.assertAlmostEqual(
+ partial_rmis_doc["scales_and_rfactors"]["r_work"],
+ 20.0 / 300.0 + 0.01,
+ places=12,
+ )
+ self.assertAlmostEqual(
+ partial_rmis_doc["scales_and_rfactors"]["replayed_r_work"],
+ 20.0 / 300.0,
+ places=12,
+ )
+ self.assertAlmostEqual(
+ partial_rmis_doc["rfactor_replay_comparison"][
+ "r_work_abs_diff"
+ ],
+ 0.01,
+ places=12,
+ )
+ self.assertIsNone(partial_rmis_doc["map_coefficients"])
+ with self.assertRaisesRegex(
+ ValueError,
+ "Partial/unverified PHENIX evidence artifact cannot be reconciled",
+ ):
+ onesar_rerun.reconcile_phenix_rfactor_output(
+ partial_rmis_doc,
+ ledger,
+ expected_vendor_source_sha256=vendor_shas,
+ expected_resolved_prefix=resolved_fake_prefix,
+ )
+
+ # 3c. Failure during map MTZ serialization/writing retains completed R-factor
+ # arrays AND already-obtained complex map_coefficients without writing out_json (#1014 Finding 4, #1017):
+ with self.subTest(case="3c_map_mtz_write_failure"):
+ out_json_mfail = tmp_dir / "rfactors_mapwrite_fail.json"
+ out_partial_mfail = (
+ tmp_dir / "rfactors_mapwrite_fail_partial_evidence.json"
+ )
+ out_diag_mfail = tmp_dir / "diagnostics_mapwrite_fail.json"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(fail_in_map_write=True),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ RuntimeError, "Injected synthetic map write failure"
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_mfail,
+ out_maps_mtz_path=tmp_dir / "mapwrite_fail_maps.mtz",
+ out_obs_pair_mtz_path=tmp_dir
+ / "mapwrite_fail_pair.mtz",
+ out_diagnostics_json_path=out_diag_mfail,
+ )
+ self.assertFalse(out_json_mfail.is_file())
+ self.assertTrue(out_partial_mfail.is_file())
+ partial_mfail_doc = json.loads(
+ out_partial_mfail.read_text(encoding="utf-8")
+ )
+ self.assertEqual(
+ partial_mfail_doc["status"], "failed_partial_unverified"
+ )
+ self.assertTrue(partial_mfail_doc["rfactor_replay_verified"])
+ self.assertFalse(partial_mfail_doc["map_export_verified"])
+ self.assertEqual(
+ partial_mfail_doc["actual_work"],
+ result_payload["actual_work"],
+ )
+ self.assertEqual(
+ partial_mfail_doc["actual_free"],
+ result_payload["actual_free"],
+ )
+ self.assertEqual(
+ partial_mfail_doc["map_coefficients"]["2mFo-DFc"],
+ result_payload["map_coefficients"]["2mFo-DFc"],
+ )
+ self.assertEqual(
+ partial_mfail_doc["map_coefficients"]["mFo-DFc"],
+ result_payload["map_coefficients"]["mFo-DFc"],
+ )
+ self.assertIsNone(
+ partial_mfail_doc["map_coefficients"]["mtz_supplements"]
+ )
+ self.assertNotIn("mtz_supplements", partial_mfail_doc)
+
+ # 3d. Failure inside fmodel.map_coefficients retains completed R-factor arrays
+ # without fabricating map_coefficients (#1014 Finding 4, #1017):
+ with self.subTest(case="3d_map_coefficients_failure"):
+ out_json_mcfail = tmp_dir / "rfactors_mapcoeff_fail.json"
+ out_partial_mcfail = (
+ tmp_dir / "rfactors_mapcoeff_fail_partial_evidence.json"
+ )
+ out_diag_mcfail = tmp_dir / "diagnostics_mapcoeff_fail.json"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(fail_in_map_coefficients=True),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ RuntimeError,
+ "Injected synthetic map_coefficients failure",
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_mcfail,
+ out_maps_mtz_path=tmp_dir
+ / "mapcoeff_fail_maps.mtz",
+ out_obs_pair_mtz_path=tmp_dir
+ / "mapcoeff_fail_pair.mtz",
+ out_diagnostics_json_path=out_diag_mcfail,
+ )
+ self.assertFalse(out_json_mcfail.is_file())
+ self.assertTrue(out_partial_mcfail.is_file())
+ partial_mcfail_doc = json.loads(
+ out_partial_mcfail.read_text(encoding="utf-8")
+ )
+ self.assertTrue(partial_mcfail_doc["rfactor_replay_verified"])
+ self.assertIsNone(partial_mcfail_doc["map_coefficients"])
+ self.assertNotIn("mtz_supplements", partial_mcfail_doc)
+
+ # 3d2. Failure on the second fmodel.map_coefficients("mFo-DFc") call retains
+ # the already obtained/exported first map ("2mFo-DFc") and completed R-factor
+ # arrays while leaving "mFo-DFc" and "mtz_supplements" None (#1014, Task 120):
+ with self.subTest(case="3d2_second_map_coefficients_failure"):
+ out_json_mc2fail = tmp_dir / "rfactors_mapcoeff2_fail.json"
+ out_partial_mc2fail = (
+ tmp_dir / "rfactors_mapcoeff2_fail_partial_evidence.json"
+ )
+ out_diag_mc2fail = tmp_dir / "diagnostics_mapcoeff2_fail.json"
+ out_maps_mc2fail = tmp_dir / "mapcoeff2_fail_maps.mtz"
+ out_pair_mc2fail = tmp_dir / "mapcoeff2_fail_pair.mtz"
+ with (
+ mock.patch.dict(
+ sys.modules,
+ _make_vendor_modules(
+ fail_in_second_map_coefficients=True
+ ),
+ ),
+ mock.patch.object(
+ onesar_phenix_measure,
+ "_inspect_vendor_modules",
+ return_value=fake_runtime_env,
+ ),
+ ):
+ with self.assertRaisesRegex(
+ RuntimeError,
+ "Injected synthetic second-map coefficient failure",
+ ):
+ onesar_phenix_measure.measure_first_block_rfactors(
+ pdb_path=pdb_path,
+ mtz_path=mtz_path,
+ params_path=params_path,
+ out_json_path=out_json_mc2fail,
+ out_maps_mtz_path=out_maps_mc2fail,
+ out_obs_pair_mtz_path=out_pair_mc2fail,
+ out_diagnostics_json_path=out_diag_mc2fail,
+ )
+ self.assertFalse(out_json_mc2fail.is_file())
+ self.assertFalse(out_maps_mc2fail.is_file())
+ self.assertFalse(out_pair_mc2fail.is_file())
+ self.assertTrue(out_partial_mc2fail.is_file())
+ partial_mc2fail_doc = json.loads(
+ out_partial_mc2fail.read_text(encoding="utf-8")
+ )
+ self.assertEqual(
+ partial_mc2fail_doc["status"], "failed_partial_unverified"
+ )
+ self.assertTrue(partial_mc2fail_doc["rfactor_replay_verified"])
+ self.assertFalse(partial_mc2fail_doc["map_export_verified"])
+ self.assertEqual(
+ partial_mc2fail_doc["extracted_pre_fmodel"],
+ result_payload["extracted_pre_fmodel"],
+ )
+ self.assertEqual(
+ partial_mc2fail_doc["final_effective"],
+ result_payload["final_effective"],
+ )
+ self.assertEqual(
+ partial_mc2fail_doc["actual_work"],
+ result_payload["actual_work"],
+ )
+ self.assertEqual(
+ partial_mc2fail_doc["actual_free"],
+ result_payload["actual_free"],
+ )
+ self.assertEqual(
+ partial_mc2fail_doc["map_coefficients"]["2mFo-DFc"],
+ result_payload["map_coefficients"]["2mFo-DFc"],
+ )
+ self.assertIsNone(
+ partial_mc2fail_doc["map_coefficients"]["mFo-DFc"]
+ )
+ self.assertIsNone(
+ partial_mc2fail_doc["map_coefficients"]["mtz_supplements"]
+ )
+ self.assertNotIn("mtz_supplements", partial_mc2fail_doc)
+ self.assertIn(
+ "map_coefficients_2mfo_dfc_raw",
+ partial_mc2fail_doc["completed_stages"],
+ )
+ self.assertNotIn(
+ "map_coefficients_raw",
+ partial_mc2fail_doc["completed_stages"],
+ )
+ with self.assertRaisesRegex(
+ ValueError,
+ "Partial/unverified PHENIX evidence artifact cannot be reconciled",
+ ):
+ onesar_rerun.reconcile_phenix_rfactor_output(
+ partial_mc2fail_doc,
+ ledger,
+ expected_vendor_source_sha256=vendor_shas,
+ expected_resolved_prefix=resolved_fake_prefix,
+ )
+
+ # 4. Direct _inspect_vendor_modules check for inside/outside prefix (#1013):
+ with self.subTest(case="4_inspect_vendor_modules_prefix"):
+ fake_bin = fake_prefix / "bin"
+ fake_bin.mkdir(parents=True, exist_ok=True)
+ fake_py = fake_bin / "python3.9"
+ fake_py.write_text("#!/bin/sh\n", encoding="utf-8")
+ inside_mod_file = fake_prefix / "lib" / "mod_inside.py"
+ inside_mod_file.parent.mkdir(parents=True, exist_ok=True)
+ inside_mod_file.write_text("# inside\n", encoding="utf-8")
+ outside_mod_file = tmp_dir / "mod_outside.py"
+ outside_mod_file.write_text("# outside\n", encoding="utf-8")
+ with (
+ mock.patch.object(sys, "prefix", resolved_fake_prefix),
+ mock.patch.object(sys, "executable", str(fake_py)),
+ ):
+ inspected = onesar_phenix_measure._inspect_vendor_modules(
+ {
+ "vendor_test": types.SimpleNamespace(
+ __name__="vendor_test",
+ __file__=str(inside_mod_file),
+ )
+ },
+ expected_resolved_prefix=resolved_fake_prefix,
+ )
+ self.assertEqual(
+ inspected["resolved_sys_prefix"], resolved_fake_prefix
+ )
+ with self.assertRaisesRegex(
+ ValueError, "escapes sys.prefix"
+ ):
+ onesar_phenix_measure._inspect_vendor_modules(
+ {
+ "vendor_test": types.SimpleNamespace(
+ __name__="vendor_test",
+ __file__=str(outside_mod_file),
+ )
+ },
+ expected_resolved_prefix=resolved_fake_prefix,
+ )
+
+ def _verify_atomic_checkpoint_interrupted_write_and_promotion(
+ self, tmp_dir: Path
+ ) -> None:
+ """Verify mid-write partial bytes and failed promotion preserve prior JSON and unlink temp siblings."""
+ atomic_target = tmp_dir / "atomic_checkpoint.json"
+ prior_payload = {
+ "status": "running",
+ "current_phase": "extract_xtal_data",
+ "phase_history": ["startup", "extract_xtal_data"],
+ "extract_xtal_data_log": "complete prior log\n",
+ }
+ onesar_phenix_measure._checkpoint_diagnostics(
+ prior_payload, atomic_target
+ )
+ prior_bytes = atomic_target.read_bytes()
+ self.assertEqual(
+ json.loads(atomic_target.read_text(encoding="utf-8")),
+ prior_payload,
+ )
+
+ real_open_temp = onesar_phenix_measure._open_checkpoint_temp_file
+ partial_temp_records: list[tuple[Path, bytes]] = []
+
+ class _InterruptedTempFileWrapper:
+ def __init__(self, tmp_path: Path) -> None:
+ self._tmp_path = tmp_path
+ self._fh = real_open_temp(tmp_path)
+
+ def __enter__(self) -> "_InterruptedTempFileWrapper":
+ self._fh.__enter__()
+ return self
+
+ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> Any:
+ return self._fh.__exit__(exc_type, exc_val, exc_tb)
+
+ def write(self, data: str) -> int:
+ partial_chunk = data[:16]
+ self._fh.write(partial_chunk)
+ self._fh.flush()
+ os.fsync(self._fh.fileno())
+ partial_temp_records.append(
+ (self._tmp_path, self._tmp_path.read_bytes())
+ )
+ raise OSError(
+ "Simulated mid-write interruption after partial bytes"
+ )
+
+ with mock.patch.object(
+ onesar_phenix_measure,
+ "_open_checkpoint_temp_file",
+ side_effect=_InterruptedTempFileWrapper,
+ ):
+ with self.assertRaisesRegex(
+ OSError,
+ "Simulated mid-write interruption after partial bytes",
+ ):
+ onesar_phenix_measure._checkpoint_diagnostics(
+ {
+ "status": "running",
+ "current_phase": "update_all_scales",
+ },
+ atomic_target,
+ )
+
+ self.assertEqual(len(partial_temp_records), 1)
+ recorded_tmp_path, recorded_partial_bytes = partial_temp_records[0]
+ self.assertEqual(len(recorded_partial_bytes), 16)
+ self.assertFalse(recorded_tmp_path.exists())
+ self.assertEqual(atomic_target.read_bytes(), prior_bytes)
+ self.assertEqual(
+ json.loads(atomic_target.read_text(encoding="utf-8")),
+ prior_payload,
+ )
+ self.assertEqual(
+ list(tmp_dir.glob(".atomic_checkpoint.json.tmp.*")), []
+ )
+
+ real_os_replace = onesar_phenix_measure.os.replace
+ replace_temp_records: list[tuple[Path, bytes]] = []
+
+ def _interrupted_os_replace(src: Any, dst: Any) -> None:
+ src_path = Path(src)
+ replace_temp_records.append((src_path, src_path.read_bytes()))
+ raise OSError("Simulated os.replace interruption")
+
+ with mock.patch.object(
+ onesar_phenix_measure.os,
+ "replace",
+ side_effect=_interrupted_os_replace,
+ ):
+ with self.assertRaisesRegex(
+ OSError, "Simulated os.replace interruption"
+ ):
+ onesar_phenix_measure._checkpoint_diagnostics(
+ {
+ "status": "running",
+ "current_phase": "update_all_scales",
+ },
+ atomic_target,
+ )
+ self.assertEqual(len(replace_temp_records), 1)
+ replace_tmp_path, replace_tmp_bytes = replace_temp_records[0]
+ self.assertEqual(
+ json.loads(replace_tmp_bytes.decode("utf-8"))["current_phase"],
+ "update_all_scales",
+ )
+ self.assertFalse(replace_tmp_path.exists())
+ self.assertEqual(atomic_target.read_bytes(), prior_bytes)
+ self.assertEqual(
+ json.loads(atomic_target.read_text(encoding="utf-8")),
+ prior_payload,
+ )
+ self.assertEqual(
+ list(tmp_dir.glob(".atomic_checkpoint.json.tmp.*")), []
+ )
+ self.assertIs(
+ onesar_phenix_measure._open_checkpoint_temp_file, real_open_temp
+ )
+ self.assertIs(onesar_phenix_measure.os.replace, real_os_replace)
+
+ def test_atomic_checkpoint_interrupted_write_and_promotion_preserve_prior_1014(
+ self,
+ ) -> None:
+ """Standalone test for mid-write partial bytes and failed promotion on atomic checkpoints."""
+ with tempfile.TemporaryDirectory() as tmp:
+ tmp_dir = Path(tmp).resolve(strict=True)
+ self._verify_atomic_checkpoint_interrupted_write_and_promotion(
+ tmp_dir
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/test_toolchain.py b/scripts/test_toolchain.py
index fc9d3ae..665cf67 100644
--- a/scripts/test_toolchain.py
+++ b/scripts/test_toolchain.py
@@ -269,4 +269,209 @@ def check(label: str, condition: bool) -> None:
and toolchain.EXTERNAL_TOOL_SPECS["gemmi"]["expected_version"] == toolchain.GEMMI_VERSION)
check("tool paths are defined only in toolchain.py", not duplicate_constants)
+with tempfile.TemporaryDirectory(prefix="refmac wrapper ; $() ") as tmp:
+ root = Path(tmp)
+ spaced_py_dir = root / "spaced python interpreter ; dir"
+ spaced_py_dir.mkdir()
+ spaced_python = spaced_py_dir / "python3"
+ spaced_python.symlink_to(Path(sys.executable))
+ fake_refmac = root / "refmac5 ; $(never-run)"
+ fake_refmac.write_text(
+ "#!/usr/bin/env python3\n"
+ "import json, os, sys\n"
+ "print(json.dumps({\n"
+ " 'argv': sys.argv,\n"
+ " 'stdin': sys.stdin.read(),\n"
+ " 'marker': os.environ.get('PROTSTRUCT_REFMAC_ENV_MARKER'),\n"
+ " 'executable': sys.executable,\n"
+ " 'path_head': os.environ.get('PATH', '').split(os.pathsep)[0],\n"
+ "}))\n"
+ )
+ fake_refmac.chmod(0o755)
+ xyzin = root / "in ; model.pdb"
+ hklin = root / "in $(obs).mtz"
+ xyzout = root / "out ; model.pdb"
+ hklout = root / "out $(obs).mtz"
+ xmlout = root / "out ; xml $(literal).xml"
+ log_path = root / "refmac ; run.log"
+ custom_env = dict(os.environ)
+ custom_env["PATH"] = f"{spaced_py_dir}{os.pathsep}{os.environ.get('PATH', '')}"
+ custom_env["PROTSTRUCT_REFMAC_ENV_MARKER"] = "explicit-ccp4-env ; $()"
+
+ ccp4_env_calls = 0
+ orig_ccp4_environment = toolchain.ccp4_environment
+ try:
+ def _stub_ccp4_environment(setup_path=toolchain.CCP4_SETUP):
+ global ccp4_env_calls
+ ccp4_env_calls += 1
+ env_copy = dict(os.environ)
+ env_copy["PATH"] = f"{spaced_py_dir}{os.pathsep}{os.environ.get('PATH', '')}"
+ env_copy["PROTSTRUCT_REFMAC_ENV_MARKER"] = "stubbed-ccp4-default-env"
+ return env_copy
+
+ toolchain.ccp4_environment = _stub_ccp4_environment
+
+ proc = toolchain.run_refmac(
+ xyzin,
+ hklin,
+ xyzout,
+ hklout,
+ log_path,
+ "NCYC 0\nEND\n",
+ cwd=root,
+ timeout=10,
+ xml_output=xmlout,
+ executable=fake_refmac,
+ env=custom_env,
+ )
+ payload = json.loads(log_path.read_text())
+ check("run_refmac with explicit executable and xml_output succeeds", proc.returncode == 0)
+ check(
+ "fake executable runs under whitespace-safe shebang via spaced interpreter PATH",
+ payload["path_head"] == str(spaced_py_dir)
+ and Path(payload["executable"]).resolve() == Path(sys.executable).resolve(),
+ )
+ check(
+ "run_refmac passes XMLOUT and metacharacter paths literally",
+ payload["argv"]
+ == [
+ str(fake_refmac),
+ "XYZIN",
+ str(xyzin),
+ "HKLIN",
+ str(hklin),
+ "XYZOUT",
+ str(xyzout),
+ "HKLOUT",
+ str(hklout),
+ "XMLOUT",
+ str(xmlout),
+ ],
+ )
+ check(
+ "run_refmac forwards stdin keywords and explicit env without calling ccp4_environment",
+ payload["stdin"] == "NCYC 0\nEND\n"
+ and payload["marker"] == "explicit-ccp4-env ; $()"
+ and ccp4_env_calls == 0,
+ )
+
+ # Explicit env with ccp4=False is also authoritative and never calls ccp4_environment
+ log_path_false = root / "refmac ; ccp4-false.log"
+ proc_false = toolchain.run_refmac(
+ xyzin,
+ hklin,
+ xyzout,
+ hklout,
+ log_path_false,
+ "NCYC 0\nEND\n",
+ cwd=root,
+ timeout=10,
+ executable=fake_refmac,
+ env=custom_env,
+ ccp4=False,
+ )
+ payload_false = json.loads(log_path_false.read_text())
+ check(
+ "run_refmac with explicit env and ccp4=False preserves supplied env",
+ proc_false.returncode == 0
+ and payload_false["marker"] == "explicit-ccp4-env ; $()"
+ and ccp4_env_calls == 0,
+ )
+
+ # Explicit env combined with ccp4=True must raise ValueError without calling ccp4_environment
+ rejected_combo = False
+ try:
+ toolchain.run_refmac(
+ xyzin,
+ hklin,
+ xyzout,
+ hklout,
+ root / "refmac ; should-not-run.log",
+ "NCYC 0\nEND\n",
+ cwd=root,
+ timeout=10,
+ executable=fake_refmac,
+ env=custom_env,
+ ccp4=True,
+ )
+ except ValueError as exc:
+ rejected_combo = "ccp4=True" in str(exc) and "env" in str(exc)
+ check(
+ "run_refmac rejects explicit env combined with ccp4=True",
+ rejected_combo
+ and ccp4_env_calls == 0
+ and not (root / "refmac ; should-not-run.log").exists(),
+ )
+
+ # Legacy neither option (env=None, ccp4=None) loads ccp4_environment()
+ log_path_legacy = root / "refmac ; legacy-ccp4.log"
+ proc_legacy = toolchain.run_refmac(
+ xyzin,
+ hklin,
+ xyzout,
+ hklout,
+ log_path_legacy,
+ "END\n",
+ cwd=root,
+ timeout=10,
+ executable=fake_refmac,
+ )
+ payload_legacy = json.loads(log_path_legacy.read_text())
+ check(
+ "legacy run_refmac without env loads ccp4_environment by default",
+ proc_legacy.returncode == 0
+ and ccp4_env_calls == 1
+ and payload_legacy["marker"] == "stubbed-ccp4-default-env",
+ )
+ finally:
+ toolchain.ccp4_environment = orig_ccp4_environment
+
+ captured_calls: list[dict[str, object]] = []
+ orig_run_logged = toolchain.run_logged
+ try:
+ def _fake_run_logged(arguments, log_p, *, cwd=None, timeout=None, env=None, ccp4=False, input_text=None):
+ captured_calls.append(
+ {
+ "arguments": list(arguments),
+ "log_path": log_p,
+ "cwd": cwd,
+ "timeout": timeout,
+ "env": env,
+ "ccp4": ccp4,
+ "input_text": input_text,
+ }
+ )
+ return toolchain.subprocess.CompletedProcess(["refmac5"], 0, "", "")
+
+ toolchain.run_logged = _fake_run_logged
+ toolchain.run_refmac(
+ "in.pdb",
+ "in.mtz",
+ "out.pdb",
+ "out.mtz",
+ "refmac.log",
+ "END\n",
+ cwd=root,
+ )
+ finally:
+ toolchain.run_logged = orig_run_logged
+ check(
+ "default run_refmac call preserves legacy argv and ccp4=True",
+ len(captured_calls) == 1
+ and captured_calls[0]["arguments"]
+ == [
+ "refmac5",
+ "XYZIN",
+ "in.pdb",
+ "HKLIN",
+ "in.mtz",
+ "XYZOUT",
+ "out.pdb",
+ "HKLOUT",
+ "out.mtz",
+ ]
+ and captured_calls[0]["ccp4"] is True
+ and captured_calls[0]["env"] is None,
+ )
+
print("\nall toolchain unit tests passed")
diff --git a/scripts/toolchain.py b/scripts/toolchain.py
index fabb6e1..1c7f2ab 100644
--- a/scripts/toolchain.py
+++ b/scripts/toolchain.py
@@ -1035,23 +1035,47 @@ def run_refmac(
*,
cwd: str | os.PathLike[str],
timeout: float = 1800,
+ xml_output: str | os.PathLike[str] | None = None,
+ executable: str | os.PathLike[str] = "refmac5",
+ env: Mapping[str, str] | None = None,
+ ccp4: bool | None = None,
) -> subprocess.CompletedProcess[str]:
- """Run REFMAC with its keyword block on stdin, without a heredoc shell."""
+ """Run REFMAC with its keyword block on stdin, without a heredoc shell.
+
+ Environment contract:
+ - When ``env`` is supplied (``env is not None``) and ``ccp4`` is omitted
+ (``None``) or ``False``, ``env`` is authoritative and ``ccp4_environment()``
+ is not called.
+ - Combining ``env is not None`` with ``ccp4=True`` raises ``ValueError`` so
+ a caller-supplied environment is never silently overwritten.
+ - Omitting both ``env`` and ``ccp4`` preserves the legacy default
+ ``ccp4=True`` (loading ``ccp4_environment()``).
+ """
+ if env is not None and ccp4:
+ raise ValueError(
+ "run_refmac cannot combine an explicit env with ccp4=True; "
+ "pass env with ccp4 omitted or False so the supplied environment is authoritative"
+ )
+ arguments: list[str | os.PathLike[str]] = [
+ executable,
+ "XYZIN",
+ model,
+ "HKLIN",
+ reflections,
+ "XYZOUT",
+ xyz_output,
+ "HKLOUT",
+ hkl_output,
+ ]
+ if xml_output is not None:
+ arguments.extend(["XMLOUT", xml_output])
+ use_ccp4 = (env is None) if ccp4 is None else bool(ccp4)
return run_logged(
- [
- "refmac5",
- "XYZIN",
- model,
- "HKLIN",
- reflections,
- "XYZOUT",
- xyz_output,
- "HKLOUT",
- hkl_output,
- ],
+ arguments,
log_path,
cwd=cwd,
timeout=timeout,
- ccp4=True,
+ env=env,
+ ccp4=use_ccp4,
input_text=keywords,
)
diff --git a/scripts/validate.sh b/scripts/validate.sh
index ab86176..df11097 100755
--- a/scripts/validate.sh
+++ b/scripts/validate.sh
@@ -216,7 +216,13 @@ if ! "${PYTHON}" "${REPO_ROOT}/scripts/test_record_tools.py"; then
fail "record tool unit tests"
fi
-# 4i. Every remaining scripts/test_*.py, discovered rather than enumerated (#228).
+# 4i. 1SAR rerun input-admission, MTZ projection, and PHENIX R-factor
+# reconciliation pure-logic unit tests (no PHENIX/CCP4/Gemmi execution).
+if ! "${PYTHON}" "${REPO_ROOT}/scripts/test_onesar_rerun.py"; then
+ fail "onesar_rerun unit tests"
+fi
+
+# 4j. Every remaining scripts/test_*.py, discovered rather than enumerated (#228).
# test_round_figures.py shipped in #188 with 18 checks and was never run here,
# because this file names each suite by hand and one was forgotten -- silently,
# as it always would be. A suite nothing executes is the repo's #2 class: a
@@ -226,7 +232,7 @@ for _suite in "${REPO_ROOT}"/scripts/test_*.py; do
case "$(basename "${_suite}")" in
test_qds_emit.py|test_t15_ss_agreement.py|test_t16_interface_quality.py| \
test_t17_nmr_ensemble.py|test_t17_restraint_summary.py|test_bench_tolerances.py| \
- test_guards.py|test_record_tools.py|test_summary_coverage.py) continue ;;
+ test_guards.py|test_record_tools.py|test_onesar_rerun.py|test_summary_coverage.py) continue ;;
esac
if ! "${PYTHON}" "${_suite}"; then
fail "$(basename "${_suite}" .py)"