From 35849c2500bbeacbb4e079c1c28d4f787a14ecb5 Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Mon, 8 Jun 2026 18:57:14 +0200 Subject: [PATCH 01/41] Add cwl-sample-displacement-transparency implementation plan --- .../cwl-sample-displacement-transparency.md | 393 ++++++++++++++++++ 1 file changed, 393 insertions(+) create mode 100644 docs/dev/plans/cwl-sample-displacement-transparency.md diff --git a/docs/dev/plans/cwl-sample-displacement-transparency.md b/docs/dev/plans/cwl-sample-displacement-transparency.md new file mode 100644 index 000000000..1360701ab --- /dev/null +++ b/docs/dev/plans/cwl-sample-displacement-transparency.md @@ -0,0 +1,393 @@ +# Plan — CWL sample-displacement & transparency peak-position corrections + +Governing conventions: [`AGENTS.md`](../../../AGENTS.md). + +**Deliberate exceptions to AGENTS.md in this plan** + +- **History.** This plan was drafted and reviewed in `tmp/plans/` + (git-ignored) to avoid interfering with in-flight work on the + `more-validation-notebooks` branch, then moved here to + `docs/dev/plans/` and committed at the start of the `/draft-impl-1` + cycle. Implementation runs on a dedicated + `cwl-sample-displacement-transparency` branch off `develop` (not on + `more-validation-notebooks`). +- **Dependency timing.** The feature depends on unreleased cryspy + functionality (PR #46). Phase 2 verification of the cryspy path can + only pass against a locally patched cryspy until that PR ships in a + released version. See *Decisions → cryspy dependency* and the + *Testing against unreleased cryspy* section. + +--- + +## Goal + +Add two FullProf-style systematic peak-position corrections to the +constant-wavelength powder instrument so the +`pd-neut-cwl_tch-fcj_lab6` verification page can be completed: + +| User-facing API | FullProf | cryspy CIF / dict key | Physical effect | 2θ shift term | +| ------------------------------------------------- | -------- | -------------------------------- | ---------------------------- | ------------------ | +| `experiment.instrument.calib_sample_displacement` | `SyCos` | `_setup_offset_SyCos` / `offset_sycos` | Specimen displacement | ∝ cos(θ) = cos(½·2θ) | +| `experiment.instrument.calib_sample_transparency` | `SySin` | `_setup_offset_SySin` / `offset_sysin` | Sample transparency/absorption | ∝ sin(2θ) | + +These join the existing `calib_twotheta_offset` (FullProf `Zero`, +cryspy `offset_ttheta`) as the third and fourth `calib_*` corrections on +`CwlPdInstrument`. + +cryspy applies all three together +(`cryspy/procedure_rhochi/rhochi_pd.py`, PR #46): + +```python +ttheta_zs = ttheta - ( + offset_ttheta + + numpy.radians(offset_sycos) * numpy.cos(0.5 * ttheta) + + numpy.radians(offset_sysin) * numpy.sin(ttheta) +) +``` + +So the EasyDiffraction parameter values are plain **degrees**, exactly +like `calib_twotheta_offset`. As the notebook already warns for `Zero`, +the FullProf-vs-cryspy sign/convention may differ, so the `.pcr` values +(`SyCos = 0.05395`, `SySin = 0.09127`) may need adjustment when wired in +— resolve this empirically in Phase 1 step P1.5. + +## Naming decision (confirmed with user) + +`calib_sample_displacement` / `calib_sample_transparency` — cause-based, +self-explanatory to non-programmer scientists, and consistent with the +`calib_*` peak-position-correction family already on the instrument +category. Chosen over `calib_sycos`/`calib_sysin` (cryptic) and over +`calib_twotheta_displacement`/`calib_twotheta_transparency` (more +verbose, no added clarity). + +## ADR + +No new ADR is required. This adds two parameters to the existing +`instrument` category following the established `Parameter` + +getter/setter + `CifHandler` pattern used by `calib_twotheta_offset` +(`src/.../instrument/cwl.py`). It introduces no new category, factory, +switchable-category wiring, or datablock. It does extend CIF +serialisation toward the cryspy backend, but only by adding rows to the +existing CWL-powder instrument mapping, which the +[`factory-contracts.md`](../adrs/accepted/factory-contracts.md) +and existing instrument design already cover. Per AGENTS.md +§Change Discipline, the relevant accepted ADRs were reviewed; none +constrains this change beyond the existing pattern. + +## Decisions + +- **Scope: CWL powder only.** The corrections are defined only for + constant-wavelength powder diffraction (cryspy restricts them to that + geometry). Add the two parameters to `CwlPdInstrument` (the + `cwl-pd` instrument), **not** to `CwlInstrumentBase` (which is shared + with the single-crystal `cwl-sc` instrument) and **not** to TOF. +- **CIF handler names.** Follow the existing `_instr.*` local + convention: `_instr.sample_displacement` and + `_instr.sample_transparency` (mirroring `_instr.2theta_offset`). +- **cryspy CIF emission.** Extend the CWL-powder branch of + `_cif_instrument_section` in + `src/easydiffraction/analysis/calculators/cryspy.py` with two rows: + `calib_sample_displacement → _setup_offset_SyCos` and + `calib_sample_transparency → _setup_offset_SySin`. +- **cryspy dict update (fast minimizer path).** Extend + `_update_experiment_in_cryspy_dict` (CWL-powder branch) to set + `cryspy_expt_dict['offset_sycos'][0]` and + `cryspy_expt_dict['offset_sysin'][0]` from the two new parameters, + mirroring how `offset_ttheta` is set. **Confirm the dict shape during + P1.3** (scalar vs `[0]`-indexed array) by inspecting the dict cryspy + builds from the emitted CIF on the PR branch; `offset_ttheta` uses + `[0]`-indexed access, so the new keys are expected to as well — verify + before committing. +- **crysfml: no support.** crysfml has no SyCos/SySin equivalent. Do + **not** add rows to `_INSTRUMENT_ATTRIBUTE_MAP` in + `crysfml.py`; `_copy_present_values` silently skips unmapped + attributes, so crysfml keeps ignoring the corrections. The notebook + already documents that crysfml will retain a systematic offset; this + is expected and acceptable. (Optionally add a one-line code comment + noting SyCos/SySin are intentionally unmapped for crysfml.) +- **cryspy dependency.** Do **not** bump the `cryspy` pin in + `pyproject.toml` in this plan: PR #46 is unreleased. The library code + is written so that, with current released cryspy (0.11.0), the new + CIF rows are simply ignored / the dict keys fall back to `0.0` + (`dict_pd.get("offset_sycos", 0.0)`), so nothing breaks. The cryspy + path of the notebook only matches FullProf once cryspy ships PR #46. + When that release exists, a **follow-up change** bumps the pin and + un-skips the page. This plan does not add a dependency under + AGENTS.md §Architecture (no pin change), so no dependency approval is + needed here. +- **Verification page un-skip is deferred.** Keep + `pd-neut-cwl_tch-fcj_lab6` in `docs/docs/verification/ci_skip.txt` + until a released cryspy supports the corrections. The notebook's two + commented `calib_*` lines get uncommented (P1.5) so the page is + ready, but it stays CI-skipped because the released engine still + differs. Note: the page is also skipped for `11B` scattering and + TCH/FCJ reasons per the current `ci_skip.txt` comment, which are + independent of this change. + +## Open questions + +1. **Sign/convention of the FullProf values.** Does + `calib_sample_displacement = 0.05395` / `calib_sample_transparency = + 0.09127` reproduce FullProf directly, or is a sign flip / scale + needed (as happened conceptually with `Zero`)? Resolve empirically in + P1.5 against a PR-#46 cryspy; record the final notebook values and + any adjustment in a code/notebook comment. +2. **cryspy dict key array shape.** Confirm `offset_sycos`/`offset_sysin` + are `[0]`-indexed arrays in the built dict (expected, matching + `offset_ttheta`). Resolved during P1.3. +3. **Released cryspy version string.** The exact released version that + first contains PR #46 is unknown today; the pin bump + un-skip is a + deferred follow-up, not part of this plan. + +## Testing against unreleased cryspy (local only — not committed) + +To exercise the cryspy path before PR #46 is released, replace the +cryspy installed in the pixi default env with the PR branch. This is a +**local developer step**; it touches `.pixi/` only and must never be +committed. + +```bash +# From a scratch dir outside the repo: +git clone --branch https://github.com/ikibalin/cryspy.git /tmp/cryspy-pr46 +# Install into the project's default pixi env (editable, so edits are live): +cd /home/andrewsazonov/Development/github.com/easyscience/diffraction-lib +.pixi/envs/default/bin/python -m pip install --no-deps -e /tmp/cryspy-pr46 +# Verify it took: +.pixi/envs/default/bin/python -c "import cryspy, inspect, os; print(os.path.dirname(cryspy.__file__))" +``` + +To restore the released cryspy afterwards (so the env matches +`pixi.lock` again): + +```bash +pixi install # or: .pixi/envs/default/bin/python -m pip install --no-deps --force-reinstall cryspy==0.11.0 +``` + +Notes: +- `` is the head branch of + https://github.com/ikibalin/cryspy/pull/46 — confirm the exact branch + name on the PR page before cloning. +- Because `pixi install` will overwrite the patched env, any + `pixi run check` / test invocation that re-syncs the env can silently + revert the patch. Run cryspy-path verification with explicit + `.pixi/envs/default/bin/python` invocations, or re-apply the patch + after any `pixi install`. +- Unit tests (P2) must **not** depend on the patched cryspy — they stub + the engine and assert on the emitted CIF string / dict mutations, per + AGENTS.md §Testing ("no real calculation engines" in unit tests). + +## Concrete files likely to change + +Source (Phase 1): + +- `src/easydiffraction/datablocks/experiment/categories/instrument/cwl.py` + — add two `Parameter`s + getter/setter properties to `CwlPdInstrument`. +- `src/easydiffraction/analysis/calculators/cryspy.py` + — `_cif_instrument_section` (CWL-powder mapping) and + `_update_experiment_in_cryspy_dict` (CWL-powder branch). +- `src/easydiffraction/analysis/calculators/crysfml.py` + — comment only (intentionally unmapped); no functional change. +- `docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py` + — uncomment the two `calib_*` lines using the new names; regenerate + the notebook with `pixi run notebook-prepare`. + +Tests (Phase 2): + +- `tests/unit/easydiffraction/datablocks/experiment/categories/instrument/test_cwl.py` + — assert the two new parameters are settable and carry correct + defaults/units/CIF names. +- A cryspy-calculator unit test (new + `test_cryspy_coverage.py` or extend an existing calculator test) + asserting the emitted CIF contains `_setup_offset_SyCos` / + `_setup_offset_SySin` with the set values, and that + `_update_experiment_in_cryspy_dict` writes `offset_sycos`/ + `offset_sysin` into a stub dict. No real engine. + +Docs / structure (auto-generated, do not hand-edit): + +- `docs/dev/package-structure/full.md`, `short.md` — regenerated by + `pixi run fix` if the public surface changes. + +## Implementation steps (Phase 1) + +> When executed via `/draft-impl-1`, each step is one atomic commit: +> stage only the files the step lists (explicit paths), update this +> checklist to `[x]` in the same commit, and use the step's `Commit:` +> message. Commit locally before moving to the next step. Do not run +> tests or `pixi run check` in Phase 1. + +- [ ] **P1.1 — Add the two parameters to `CwlPdInstrument`.** + In `cwl.py`, inside `CwlPdInstrument.__init__`, add + `self._calib_sample_displacement` and + `self._calib_sample_transparency` `Parameter`s modeled on + `_calib_twotheta_offset`: `name='sample_displacement'` / + `'sample_transparency'`, descriptive `description`, `units='degrees'`, + `DisplayHandler` (display names "Sample displacement" / "Sample + transparency", `display_units='deg'`, sensible LaTeX), default `0.0`, + `RangeValidator()`, and `CifHandler(names=['_instr.sample_displacement'])` + / `['_instr.sample_transparency']`. Add the matching getter + + setter properties (numpy-style one-line ≤72-char docstrings). + Files: `src/.../instrument/cwl.py`. + Commit: `Add CWL sample displacement and transparency parameters` + +- [ ] **P1.2 — Emit the corrections in the cryspy CIF.** + In `cryspy.py` `_cif_instrument_section`, CWL+powder branch, extend + `instrument_mapping` with + `'calib_sample_displacement': '_setup_offset_SyCos'` and + `'calib_sample_transparency': '_setup_offset_SySin'`. + Files: `src/easydiffraction/analysis/calculators/cryspy.py`. + Commit: `Emit SyCos/SySin offsets in cryspy CWL instrument CIF` + +- [ ] **P1.3 — Update the cached cryspy dict (fast path).** + In `cryspy.py` `_update_experiment_in_cryspy_dict`, CWL+powder + branch, set `cryspy_expt_dict['offset_sycos'][0]` and + `cryspy_expt_dict['offset_sysin'][0]` from the two new parameters, + next to the existing `offset_ttheta` assignment. First confirm the + built dict exposes these keys with `[0]`-indexed shape (see Open + question 2); guard with `if 'offset_sycos' in cryspy_expt_dict:` only + if released cryspy lacks the keys and would otherwise KeyError on the + minimizer fast path — prefer the unconditional form if released + cryspy already provides zero-initialized keys. + Files: `src/easydiffraction/analysis/calculators/cryspy.py`. + Commit: `Update cryspy dict with SyCos/SySin offsets` + +- [ ] **P1.4 — Document crysfml non-support.** + In `crysfml.py` `_update_experiment_dict_from_instrument`, add a + one-line comment near the instrument map noting SyCos/SySin + (`calib_sample_displacement` / `calib_sample_transparency`) are + intentionally unmapped because crysfml has no equivalent correction. + No functional change. + Files: `src/easydiffraction/analysis/calculators/crysfml.py`. + Commit: `Note crysfml lacks SyCos/SySin instrument corrections` + +- [ ] **P1.5 — Wire the corrections into the verification page.** + In `pd-neut-cwl_tch-fcj_lab6.py`, uncomment the two correction lines + using the new names and the empirically confirmed values + (start from `calib_sample_displacement = 0.05395`, + `calib_sample_transparency = 0.09127`; adjust per Open question 1 if + the cryspy convention differs). Update the surrounding markdown so it + no longer says "pending EasyDiffraction support". Keep the page in + `ci_skip.txt` (still skipped until released cryspy supports PR #46); + update only the cryspy-path narrative if needed. Regenerate the + notebook with `pixi run notebook-prepare`. + Files: `docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py`, + `docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb`. + Commit: `Enable SyCos/SySin corrections in LaB6 verification page` + +- [ ] **P1.6 — Phase 1 review gate** (no-code). Mark `[x]`, commit the + checklist update alone. + Commit: `Reach Phase 1 review gate` + +## Implementation steps (Phase 2) + +> Run after Phase 1 review is approved. When executed via +> `/draft-impl-2`, **add/extend the tests first (P2.1)**, then run the +> verification suite (P2.2–P2.6) in order, committing atomically per +> auto-fix or per logical fix. Use the zsh-safe log-capture pattern +> below whenever a task's output must be analyzed. Do not silence lint +> thresholds (`# noqa`, threshold bumps) — refactor instead per +> AGENTS.md §Code Style. + +- [ ] **P2.1 — Add/extend unit tests (engine-free).** + - Extend + `tests/unit/.../instrument/test_cwl.py` to assert + `calib_sample_displacement` / `calib_sample_transparency` are + settable, default to `0.0`, carry `units='degrees'`, and expose CIF + names `_instr.sample_displacement` / `_instr.sample_transparency`. + - Add a cryspy-calculator unit test (new + `tests/unit/.../analysis/calculators/test_cryspy_coverage.py` or + extend an existing calculator test) asserting that + `_cif_instrument_section` emits `_setup_offset_SyCos` / + `_setup_offset_SySin` with the set values, and that + `_update_experiment_in_cryspy_dict` writes `offset_sycos` / + `offset_sysin` into a stub experiment dict. **No real cryspy + engine** (AGENTS.md §Testing); the unreleased PR-#46 cryspy is + exercised manually per *Testing against unreleased cryspy*, + outside `pixi run unit-tests`. + - Confirm test/source mirroring with `pixi run test-structure-check` + (also run inside `pixi run check`). + - Commit: `Add tests for CWL SyCos/SySin instrument corrections` + +- [ ] **P2.2 — `pixi run fix`.** Apply auto-fixes; include any + regenerated `docs/dev/package-structure/full.md` / `short.md`. + Commit: `Apply pixi run fix auto-fixes` (skip if nothing changed). + +- [ ] **P2.3 — `pixi run check`** until clean. Fix mechanical lint + nits directly; refactor (don't silence) any complexity/type breach. + ```bash + pixi run check > /tmp/easydiffraction-check.log 2>&1; check_exit_code=$?; tail -n 200 /tmp/easydiffraction-check.log; exit $check_exit_code + ``` + +- [ ] **P2.4 — `pixi run unit-tests`** until green. + ```bash + pixi run unit-tests > /tmp/easydiffraction-unit.log 2>&1; unit_tests_exit_code=$?; tail -n 200 /tmp/easydiffraction-unit.log; exit $unit_tests_exit_code + ``` + +- [ ] **P2.5 — `pixi run integration-tests`** until green. For + sandbox-only multiprocessing/process-pool failures, rerun with the + approved escalated permission path before treating it as a defect. + ```bash + pixi run integration-tests + ``` + +- [ ] **P2.6 — `pixi run script-tests`** until green. + `pd-neut-cwl_tch-fcj_lab6` remains skipped via `ci_skip.txt`, so the + still-divergent cryspy path will not fail CI; confirm the page still + parses/builds where the runner loads it. Leave generated + `docs/dev/benchmarking/*.csv` untracked. + ```bash + pixi run script-tests > /tmp/easydiffraction-script.log 2>&1; script_tests_exit_code=$?; tail -n 200 /tmp/easydiffraction-script.log; exit $script_tests_exit_code + ``` + +## Follow-up (out of scope, deferred) + +- When a released cryspy includes PR #46: bump the `cryspy` pin in + `pyproject.toml` / refresh `pixi.lock`, remove + `pd-neut-cwl_tch-fcj_lab6` from `ci_skip.txt` (if the remaining + `11B` / TCH-FCJ discrepancies are also resolved), and finalize the + notebook reference values. Track in `docs/dev/issues/open.md`. + +## Status checklist + +- [ ] P1.1 Add parameters to `CwlPdInstrument` +- [ ] P1.2 Emit offsets in cryspy CIF +- [ ] P1.3 Update cryspy cached dict +- [ ] P1.4 Comment crysfml non-support +- [ ] P1.5 Wire corrections into verification page +- [ ] P1.6 Phase 1 review gate +- [ ] P2.1 Add/extend engine-free unit tests +- [ ] P2.2 `pixi run fix` +- [ ] P2.3 `pixi run check` clean +- [ ] P2.4 `pixi run unit-tests` green +- [ ] P2.5 `pixi run integration-tests` green +- [ ] P2.6 `pixi run script-tests` green + +## Suggested branch + +`cwl-sample-displacement-transparency` (off `develop`, per AGENTS.md +§Planning). Note: current work is on `more-validation-notebooks`; create +the implementation branch when starting `/draft-impl-1`. + +## Suggested Pull Request + +**Title:** Add sample-displacement and transparency peak corrections for +constant-wavelength powder data + +**Description:** Constant-wavelength powder instruments can now correct +for two common sources of systematic peak-position error: **sample +displacement** (the specimen sitting slightly off the diffractometer +axis) and **sample transparency** (the beam penetrating into the +sample). These match FullProf's `SyCos` and `SySin` corrections and sit +alongside the existing zero-offset correction: + +```python +experiment.instrument.calib_sample_displacement = 0.05395 +experiment.instrument.calib_sample_transparency = 0.09127 +``` + +This lets EasyDiffraction reproduce FullProf peak positions for datasets +where these effects matter (e.g. the LaB₆ verification reference) when +using the cryspy engine. The crysfml engine does not yet support these +corrections. Until the upstream cryspy release that adds them is +available, the LaB₆ verification page remains marked as a known +work-in-progress and is excluded from CI. From 487014007eab133c393542552f5b25ed3e8930ed Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Mon, 8 Jun 2026 18:58:26 +0200 Subject: [PATCH 02/41] Add CWL sample displacement and transparency parameters --- .../cwl-sample-displacement-transparency.md | 4 +- .../experiment/categories/instrument/cwl.py | 72 +++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/docs/dev/plans/cwl-sample-displacement-transparency.md b/docs/dev/plans/cwl-sample-displacement-transparency.md index 1360701ab..e445b4674 100644 --- a/docs/dev/plans/cwl-sample-displacement-transparency.md +++ b/docs/dev/plans/cwl-sample-displacement-transparency.md @@ -216,7 +216,7 @@ Docs / structure (auto-generated, do not hand-edit): > message. Commit locally before moving to the next step. Do not run > tests or `pixi run check` in Phase 1. -- [ ] **P1.1 — Add the two parameters to `CwlPdInstrument`.** +- [x] **P1.1 — Add the two parameters to `CwlPdInstrument`.** In `cwl.py`, inside `CwlPdInstrument.__init__`, add `self._calib_sample_displacement` and `self._calib_sample_transparency` `Parameter`s modeled on @@ -349,7 +349,7 @@ Docs / structure (auto-generated, do not hand-edit): ## Status checklist -- [ ] P1.1 Add parameters to `CwlPdInstrument` +- [x] P1.1 Add parameters to `CwlPdInstrument` - [ ] P1.2 Emit offsets in cryspy CIF - [ ] P1.3 Update cryspy cached dict - [ ] P1.4 Comment crysfml non-support diff --git a/src/easydiffraction/datablocks/experiment/categories/instrument/cwl.py b/src/easydiffraction/datablocks/experiment/categories/instrument/cwl.py index 757e379f5..452e10477 100644 --- a/src/easydiffraction/datablocks/experiment/categories/instrument/cwl.py +++ b/src/easydiffraction/datablocks/experiment/categories/instrument/cwl.py @@ -130,6 +130,48 @@ def __init__(self) -> None: ), ) + self._calib_sample_displacement: Parameter = Parameter( + name='sample_displacement', + description='Specimen displacement from the diffractometer axis', + units='degrees', + display_handler=DisplayHandler( + display_name='Sample displacement', + display_units='deg', + latex_name='Sample displacement', + latex_units=r'\mathrm{deg}', + ), + value_spec=AttributeSpec( + default=0.0, + validator=RangeValidator(), + ), + cif_handler=CifHandler( + names=[ + '_instr.sample_displacement', + ] + ), + ) + + self._calib_sample_transparency: Parameter = Parameter( + name='sample_transparency', + description='Sample transparency (beam penetration) shift', + units='degrees', + display_handler=DisplayHandler( + display_name='Sample transparency', + display_units='deg', + latex_name='Sample transparency', + latex_units=r'\mathrm{deg}', + ), + value_spec=AttributeSpec( + default=0.0, + validator=RangeValidator(), + ), + cif_handler=CifHandler( + names=[ + '_instr.sample_transparency', + ] + ), + ) + @property def calib_twotheta_offset(self) -> Parameter: """ @@ -144,3 +186,33 @@ def calib_twotheta_offset(self) -> Parameter: def calib_twotheta_offset(self, value: float) -> None: """Set the instrument misalignment offset (deg).""" self._calib_twotheta_offset.value = value + + @property + def calib_sample_displacement(self) -> Parameter: + """ + Specimen-displacement peak-position correction (deg). + + Reading this property returns the underlying ``Parameter`` + object. Assigning to it updates the parameter value. + """ + return self._calib_sample_displacement + + @calib_sample_displacement.setter + def calib_sample_displacement(self, value: float) -> None: + """Set the specimen-displacement correction (deg).""" + self._calib_sample_displacement.value = value + + @property + def calib_sample_transparency(self) -> Parameter: + """ + Sample-transparency peak-position correction (deg). + + Reading this property returns the underlying ``Parameter`` + object. Assigning to it updates the parameter value. + """ + return self._calib_sample_transparency + + @calib_sample_transparency.setter + def calib_sample_transparency(self, value: float) -> None: + """Set the sample-transparency correction (deg).""" + self._calib_sample_transparency.value = value From ee0b833964a9c6331ede6c10017d088ae73cbd83 Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Mon, 8 Jun 2026 18:58:50 +0200 Subject: [PATCH 03/41] Emit SyCos/SySin offsets in cryspy CWL instrument CIF --- docs/dev/plans/cwl-sample-displacement-transparency.md | 4 ++-- src/easydiffraction/analysis/calculators/cryspy.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/dev/plans/cwl-sample-displacement-transparency.md b/docs/dev/plans/cwl-sample-displacement-transparency.md index e445b4674..ab3c249d5 100644 --- a/docs/dev/plans/cwl-sample-displacement-transparency.md +++ b/docs/dev/plans/cwl-sample-displacement-transparency.md @@ -230,7 +230,7 @@ Docs / structure (auto-generated, do not hand-edit): Files: `src/.../instrument/cwl.py`. Commit: `Add CWL sample displacement and transparency parameters` -- [ ] **P1.2 — Emit the corrections in the cryspy CIF.** +- [x] **P1.2 — Emit the corrections in the cryspy CIF.** In `cryspy.py` `_cif_instrument_section`, CWL+powder branch, extend `instrument_mapping` with `'calib_sample_displacement': '_setup_offset_SyCos'` and @@ -350,7 +350,7 @@ Docs / structure (auto-generated, do not hand-edit): ## Status checklist - [x] P1.1 Add parameters to `CwlPdInstrument` -- [ ] P1.2 Emit offsets in cryspy CIF +- [x] P1.2 Emit offsets in cryspy CIF - [ ] P1.3 Update cryspy cached dict - [ ] P1.4 Comment crysfml non-support - [ ] P1.5 Wire corrections into verification page diff --git a/src/easydiffraction/analysis/calculators/cryspy.py b/src/easydiffraction/analysis/calculators/cryspy.py index 2670f89e4..77ba22d38 100644 --- a/src/easydiffraction/analysis/calculators/cryspy.py +++ b/src/easydiffraction/analysis/calculators/cryspy.py @@ -941,6 +941,8 @@ def _cif_instrument_section( instrument_mapping = { 'setup_wavelength': '_setup_wavelength', 'calib_twotheta_offset': '_setup_offset_2theta', + 'calib_sample_displacement': '_setup_offset_SyCos', + 'calib_sample_transparency': '_setup_offset_SySin', } elif expt_type.sample_form.value == SampleFormEnum.SINGLE_CRYSTAL: instrument_mapping = {'setup_wavelength': '_setup_wavelength'} From 0679b4febbca690db34c935731af7befc756a667 Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Mon, 8 Jun 2026 19:00:30 +0200 Subject: [PATCH 04/41] Update cryspy dict with SyCos/SySin offsets --- .../plans/cwl-sample-displacement-transparency.md | 4 ++-- .../analysis/calculators/cryspy.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/dev/plans/cwl-sample-displacement-transparency.md b/docs/dev/plans/cwl-sample-displacement-transparency.md index ab3c249d5..4f3add8b9 100644 --- a/docs/dev/plans/cwl-sample-displacement-transparency.md +++ b/docs/dev/plans/cwl-sample-displacement-transparency.md @@ -238,7 +238,7 @@ Docs / structure (auto-generated, do not hand-edit): Files: `src/easydiffraction/analysis/calculators/cryspy.py`. Commit: `Emit SyCos/SySin offsets in cryspy CWL instrument CIF` -- [ ] **P1.3 — Update the cached cryspy dict (fast path).** +- [x] **P1.3 — Update the cached cryspy dict (fast path).** In `cryspy.py` `_update_experiment_in_cryspy_dict`, CWL+powder branch, set `cryspy_expt_dict['offset_sycos'][0]` and `cryspy_expt_dict['offset_sysin'][0]` from the two new parameters, @@ -351,7 +351,7 @@ Docs / structure (auto-generated, do not hand-edit): - [x] P1.1 Add parameters to `CwlPdInstrument` - [x] P1.2 Emit offsets in cryspy CIF -- [ ] P1.3 Update cryspy cached dict +- [x] P1.3 Update cryspy cached dict - [ ] P1.4 Comment crysfml non-support - [ ] P1.5 Wire corrections into verification page - [ ] P1.6 Phase 1 review gate diff --git a/src/easydiffraction/analysis/calculators/cryspy.py b/src/easydiffraction/analysis/calculators/cryspy.py index 77ba22d38..1c1cd7fd7 100644 --- a/src/easydiffraction/analysis/calculators/cryspy.py +++ b/src/easydiffraction/analysis/calculators/cryspy.py @@ -647,6 +647,21 @@ def _update_experiment_in_cryspy_dict( ) cryspy_expt_dict['wavelength'][0] = experiment.instrument.setup_wavelength.value + # Sample-displacement (SyCos) and transparency (SySin) + # peak-position corrections (cryspy PR #46). cryspy + # applies numpy.radians() to these internally, so the + # dict stores plain degrees here (unlike offset_ttheta, + # which is pre-converted to radians). The keys are absent + # on cryspy releases without PR #46, so guard before set. + if 'offset_sycos' in cryspy_expt_dict: + cryspy_expt_dict['offset_sycos'][0] = ( + experiment.instrument.calib_sample_displacement.value + ) + if 'offset_sysin' in cryspy_expt_dict: + cryspy_expt_dict['offset_sysin'][0] = ( + experiment.instrument.calib_sample_transparency.value + ) + # Peak cryspy_resolution = cryspy_expt_dict['resolution_parameters'] cryspy_resolution[0] = experiment.peak.broad_gauss_u.value From 44663389fd2704876d2d4cce883bbb796d382cdc Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Mon, 8 Jun 2026 19:01:04 +0200 Subject: [PATCH 05/41] Note crysfml lacks SyCos/SySin instrument corrections --- docs/dev/plans/cwl-sample-displacement-transparency.md | 4 ++-- src/easydiffraction/analysis/calculators/crysfml.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/dev/plans/cwl-sample-displacement-transparency.md b/docs/dev/plans/cwl-sample-displacement-transparency.md index 4f3add8b9..da5af1c4a 100644 --- a/docs/dev/plans/cwl-sample-displacement-transparency.md +++ b/docs/dev/plans/cwl-sample-displacement-transparency.md @@ -251,7 +251,7 @@ Docs / structure (auto-generated, do not hand-edit): Files: `src/easydiffraction/analysis/calculators/cryspy.py`. Commit: `Update cryspy dict with SyCos/SySin offsets` -- [ ] **P1.4 — Document crysfml non-support.** +- [x] **P1.4 — Document crysfml non-support.** In `crysfml.py` `_update_experiment_dict_from_instrument`, add a one-line comment near the instrument map noting SyCos/SySin (`calib_sample_displacement` / `calib_sample_transparency`) are @@ -352,7 +352,7 @@ Docs / structure (auto-generated, do not hand-edit): - [x] P1.1 Add parameters to `CwlPdInstrument` - [x] P1.2 Emit offsets in cryspy CIF - [x] P1.3 Update cryspy cached dict -- [ ] P1.4 Comment crysfml non-support +- [x] P1.4 Comment crysfml non-support - [ ] P1.5 Wire corrections into verification page - [ ] P1.6 Phase 1 review gate - [ ] P2.1 Add/extend engine-free unit tests diff --git a/src/easydiffraction/analysis/calculators/crysfml.py b/src/easydiffraction/analysis/calculators/crysfml.py index 34e8d0cb1..9fc182a46 100644 --- a/src/easydiffraction/analysis/calculators/crysfml.py +++ b/src/easydiffraction/analysis/calculators/crysfml.py @@ -37,6 +37,9 @@ _INSTRUMENT_ATTRIBUTE_MAP: tuple[tuple[str, str], ...] = ( ('setup_wavelength', '_diffrn_radiation_wavelength'), ('calib_twotheta_offset', '_pd_meas_2theta_offset'), + # crysfml has no SyCos/SySin equivalent, so the CWL + # calib_sample_displacement and calib_sample_transparency + # corrections are intentionally left unmapped here. ('calib_d_to_tof_offset', '_pd_meas_tof_offset'), ('calib_d_to_tof_linear', '_pd_meas_tof_dtt1'), ('calib_d_to_tof_quad', '_pd_meas_tof_dtt2'), From a66bd3feb7bee139a4f8c553c90459f25a4cddf9 Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Mon, 8 Jun 2026 19:02:28 +0200 Subject: [PATCH 06/41] Enable SyCos/SySin corrections in LaB6 verification page --- .../cwl-sample-displacement-transparency.md | 4 ++-- .../pd-neut-cwl_tch-fcj_lab6.ipynb | 22 +++++++++++-------- .../verification/pd-neut-cwl_tch-fcj_lab6.py | 22 +++++++++++-------- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/docs/dev/plans/cwl-sample-displacement-transparency.md b/docs/dev/plans/cwl-sample-displacement-transparency.md index da5af1c4a..2a25addeb 100644 --- a/docs/dev/plans/cwl-sample-displacement-transparency.md +++ b/docs/dev/plans/cwl-sample-displacement-transparency.md @@ -260,7 +260,7 @@ Docs / structure (auto-generated, do not hand-edit): Files: `src/easydiffraction/analysis/calculators/crysfml.py`. Commit: `Note crysfml lacks SyCos/SySin instrument corrections` -- [ ] **P1.5 — Wire the corrections into the verification page.** +- [x] **P1.5 — Wire the corrections into the verification page.** In `pd-neut-cwl_tch-fcj_lab6.py`, uncomment the two correction lines using the new names and the empirically confirmed values (start from `calib_sample_displacement = 0.05395`, @@ -353,7 +353,7 @@ Docs / structure (auto-generated, do not hand-edit): - [x] P1.2 Emit offsets in cryspy CIF - [x] P1.3 Update cryspy cached dict - [x] P1.4 Comment crysfml non-support -- [ ] P1.5 Wire corrections into verification page +- [x] P1.5 Wire corrections into verification page - [ ] P1.6 Phase 1 review gate - [ ] P2.1 Add/extend engine-free unit tests - [ ] P2.2 `pixi run fix` diff --git a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb index e285b2fae..51b1d8c80 100644 --- a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb +++ b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb @@ -177,15 +177,19 @@ "id": "11", "metadata": {}, "source": [ - "## SyCos / SySin (pending EasyDiffraction support)\n", + "## Sample displacement / transparency (SyCos / SySin)\n", "\n", "FullProf applies sample-displacement (`SyCos`) and transparency\n", - "(`SySin`) peak-position shifts on top of `Zero` (see issue #117). The\n", - "CWL instrument category does not expose them yet, so the two lines\n", - "below are kept commented out with the FullProf `.pcr` values —\n", - "uncomment them once the parameters land to finish this page. As with\n", - "`Zero` (`calib_twotheta_offset` above), the cross-code convention may\n", - "differ, so the values may need the same adjustment when wired in." + "(`SySin`) peak-position shifts on top of `Zero` (see issue #117).\n", + "These map to `calib_sample_displacement` and\n", + "`calib_sample_transparency` on the CWL powder instrument. The cryspy\n", + "engine applies them only with the new functionality from\n", + "[cryspy PR #46](https://github.com/ikibalin/cryspy/pull/46); on the\n", + "currently released cryspy the corrections are ignored, and crysfml has\n", + "no equivalent, so this page stays in `ci_skip.txt` until a cryspy\n", + "release ships the support. As with `Zero` (`calib_twotheta_offset`\n", + "above), the cross-code convention may differ, so these `.pcr` values\n", + "may still need adjustment once validated against a PR #46 cryspy." ] }, { @@ -195,8 +199,8 @@ "metadata": {}, "outputs": [], "source": [ - "# experiment.instrument.calib_sycos = 0.05395 # FullProf SyCos\n", - "# experiment.instrument.calib_sysin = 0.09127 # FullProf SySin" + "experiment.instrument.calib_sample_displacement = 0.05395 # FullProf SyCos\n", + "experiment.instrument.calib_sample_transparency = 0.09127 # FullProf SySin" ] }, { diff --git a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py index 4f6cb7bed..83907b16a 100644 --- a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py +++ b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py @@ -93,19 +93,23 @@ project.experiments.add(experiment) # %% [markdown] -# ## SyCos / SySin (pending EasyDiffraction support) +# ## Sample displacement / transparency (SyCos / SySin) # # FullProf applies sample-displacement (`SyCos`) and transparency -# (`SySin`) peak-position shifts on top of `Zero` (see issue #117). The -# CWL instrument category does not expose them yet, so the two lines -# below are kept commented out with the FullProf `.pcr` values — -# uncomment them once the parameters land to finish this page. As with -# `Zero` (`calib_twotheta_offset` above), the cross-code convention may -# differ, so the values may need the same adjustment when wired in. +# (`SySin`) peak-position shifts on top of `Zero` (see issue #117). +# These map to `calib_sample_displacement` and +# `calib_sample_transparency` on the CWL powder instrument. The cryspy +# engine applies them only with the new functionality from +# [cryspy PR #46](https://github.com/ikibalin/cryspy/pull/46); on the +# currently released cryspy the corrections are ignored, and crysfml has +# no equivalent, so this page stays in `ci_skip.txt` until a cryspy +# release ships the support. As with `Zero` (`calib_twotheta_offset` +# above), the cross-code convention may differ, so these `.pcr` values +# may still need adjustment once validated against a PR #46 cryspy. # %% -# experiment.instrument.calib_sycos = 0.05395 # FullProf SyCos -# experiment.instrument.calib_sysin = 0.09127 # FullProf SySin +experiment.instrument.calib_sample_displacement = 0.05395 # FullProf SyCos +experiment.instrument.calib_sample_transparency = 0.09127 # FullProf SySin # %% [markdown] # ## Calculate the pattern with each engine From 6dac9a1fdf9539fd16d3697d0cb37cc7375bad24 Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Mon, 8 Jun 2026 19:02:54 +0200 Subject: [PATCH 07/41] Reach Phase 1 review gate --- docs/dev/plans/cwl-sample-displacement-transparency.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/plans/cwl-sample-displacement-transparency.md b/docs/dev/plans/cwl-sample-displacement-transparency.md index 2a25addeb..ceaadf24a 100644 --- a/docs/dev/plans/cwl-sample-displacement-transparency.md +++ b/docs/dev/plans/cwl-sample-displacement-transparency.md @@ -274,7 +274,7 @@ Docs / structure (auto-generated, do not hand-edit): `docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb`. Commit: `Enable SyCos/SySin corrections in LaB6 verification page` -- [ ] **P1.6 — Phase 1 review gate** (no-code). Mark `[x]`, commit the +- [x] **P1.6 — Phase 1 review gate** (no-code). Mark `[x]`, commit the checklist update alone. Commit: `Reach Phase 1 review gate` @@ -354,7 +354,7 @@ Docs / structure (auto-generated, do not hand-edit): - [x] P1.3 Update cryspy cached dict - [x] P1.4 Comment crysfml non-support - [x] P1.5 Wire corrections into verification page -- [ ] P1.6 Phase 1 review gate +- [x] P1.6 Phase 1 review gate - [ ] P2.1 Add/extend engine-free unit tests - [ ] P2.2 `pixi run fix` - [ ] P2.3 `pixi run check` clean From 996efc6bee405d7a015c26a5eafc685fc5f5b3f7 Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Mon, 8 Jun 2026 20:15:24 +0200 Subject: [PATCH 08/41] Strip isotope number from atom symbol for crysfml --- .../analysis/calculators/crysfml.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/easydiffraction/analysis/calculators/crysfml.py b/src/easydiffraction/analysis/calculators/crysfml.py index 9fc182a46..790ed9d48 100644 --- a/src/easydiffraction/analysis/calculators/crysfml.py +++ b/src/easydiffraction/analysis/calculators/crysfml.py @@ -71,6 +71,27 @@ ) +def _element_symbol(type_symbol: str) -> str: + """ + Strip a leading isotope number from an atom type symbol. + + CrysFML resolves scattering by element and does not understand + isotope prefixes such as ``11B`` or ``2H`` (cryspy does). Returning + the bare element symbol lets one model drive both engines. + + Parameters + ---------- + type_symbol : str + Atom type symbol, optionally isotope-prefixed (e.g. ``11B``). + + Returns + ------- + str + The symbol with any leading digits removed (e.g. ``B``). + """ + return type_symbol.lstrip('0123456789') + + @CalculatorFactory.register class CrysfmlCalculator(CalculatorBase): """Wrapper for Crysfml library.""" @@ -265,7 +286,7 @@ def _convert_structure_to_dict( # noqa: PLR6301 for atom in structure.atom_sites: atom_site = { '_label': atom.label.value, - '_type_symbol': atom.type_symbol.value, + '_type_symbol': _element_symbol(atom.type_symbol.value), '_fract_x': atom.fract_x.value, '_fract_y': atom.fract_y.value, '_fract_z': atom.fract_z.value, From 0d3949229d91460432b96eb6787195b4acceece6 Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Mon, 8 Jun 2026 20:24:04 +0200 Subject: [PATCH 09/41] Refine SyCos/SySin and use 11B in LaB6 verification page --- docs/docs/verification/ci_skip.txt | 2 +- .../pd-neut-cwl_tch-fcj_lab6.ipynb | 100 +++++++++++------- .../verification/pd-neut-cwl_tch-fcj_lab6.py | 91 ++++++++++------ 3 files changed, 120 insertions(+), 73 deletions(-) diff --git a/docs/docs/verification/ci_skip.txt b/docs/docs/verification/ci_skip.txt index 9da54f03d..8869f6498 100644 --- a/docs/docs/verification/ci_skip.txt +++ b/docs/docs/verification/ci_skip.txt @@ -14,5 +14,5 @@ # # Example (do not leave commented examples as active entries): pd-neut-cwl_pv-asym_empir_pbso4 # Asymmetry params are different -pd-neut-cwl_tch-fcj_lab6 # SyCos/SySin, 11B scattering, TCH/FCJ (cryspy PR #46) +pd-neut-cwl_tch-fcj_lab6 # needs unreleased cryspy PR #46 (SyCos/SySin); residual TCH/FCJ profile diff pd-neut-tof_jvd_si # cryspy TOF Lorentzian discrepancy; see issues/open.md diff --git a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb index 51b1d8c80..d315b5e3e 100644 --- a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb +++ b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb @@ -26,10 +26,25 @@ "source": [ "# LaB₆ — neutron powder, constant wavelength, Thompson–Cox–Hastings\n", "\n", - "A **prepared** verification for the FullProf `SyCos`/`SySin` systematic\n", - "peak-position corrections (sample displacement and transparency), using\n", - "the real LaB6 dataset from\n", - "[cryspy issue #38](https://github.com/ikibalin/cryspy/issues/38).\n" + "This page verifies the FullProf `SyCos`/`SySin` systematic\n", + "peak-position corrections (sample displacement and transparency) for a\n", + "constant-wavelength powder experiment, using the real LaB₆ dataset from\n", + "[cryspy issue #38](https://github.com/ikibalin/cryspy/issues/38).\n", + "\n", + "`SyCos`/`SySin` map to `calib_sample_displacement` and\n", + "`calib_sample_transparency` on the CWL powder instrument. Only the\n", + "`cryspy` engine applies them, and only with the functionality added in\n", + "[cryspy PR #46](https://github.com/ikibalin/cryspy/pull/46); the\n", + "`crysfml` engine has no equivalent. Because FullProf and cryspy use\n", + "**different coefficient conventions** for these corrections (and a\n", + "different absolute-intensity scale), the `.pcr` values are used only as\n", + "starting points: the page **refines** `scale`, `calib_sample_displacement`\n", + "and `calib_sample_transparency` against the FullProf profile with cryspy,\n", + "then reuses the refined `scale` for crysfml (which keeps a peak-position\n", + "offset, since it cannot apply the corrections).\n", + "\n", + "The page stays listed in `ci_skip.txt` until a released cryspy ships the\n", + "PR #46 corrections." ] }, { @@ -96,7 +111,11 @@ "id": "7", "metadata": {}, "source": [ - "## Define the structure" + "## Define the structure\n", + "\n", + "Boron is the ¹¹B isotope (FullProf `B11`). The `cryspy` engine resolves\n", + "the isotope directly; the `crysfml` engine resolves scattering by\n", + "element and silently drops the isotope number (`11B` → `B`)." ] }, { @@ -116,14 +135,11 @@ " fract_y=0.0, # FullProf Y\n", " fract_z=0.0, # FullProf Z\n", " adp_type='Biso', # FullProf Biso\n", - " adp_iso=0.53405, # FullProf Biso\n", + " adp_iso=0.53399, # FullProf Biso\n", ")\n", "structure.atom_sites.create(\n", " label='B', # FullProf Atom\n", - " # ❌ returned a\n", - " # result with an exception set\n", - " # type_symbol='11B', # FullProf \"B11 0.66500 0.00000 0\"\n", - " type_symbol='B', # FullProf \"B11 0.66500 0.00000 0\"\n", + " type_symbol='11B', # FullProf \"B11 0.66500 0.00000 0\"\n", " fract_x=0.19972, # FullProf X\n", " fract_y=0.5, # FullProf Y\n", " fract_z=0.5, # FullProf Z\n", @@ -139,7 +155,11 @@ "id": "9", "metadata": {}, "source": [ - "## Create the experiment" + "## Create the experiment\n", + "\n", + "Starting values are taken from `ECH0030684_LaB6_1p622A.pcr`. The\n", + "`SyCos`/`SySin` values are FullProf-convention starting points and are\n", + "refined below into cryspy's equivalents." ] }, { @@ -158,16 +178,18 @@ ")\n", "verify.set_reference_as_measured(experiment, x, calc_fullprof)\n", "\n", - "experiment.linked_phases.create(id='lab6', scale=136.0509) # FullProf Scale\n", + "experiment.linked_phases.create(id='lab6', scale=136.0507) # FullProf Scale\n", "\n", "experiment.instrument.setup_wavelength = 1.623891 # FullProf Lambda\n", - "experiment.instrument.calib_twotheta_offset = -0.45497 # FullProf Zero\n", + "experiment.instrument.calib_twotheta_offset = -0.45501 # FullProf Zero\n", + "experiment.instrument.calib_sample_displacement = 0.01052 # FullProf SyCos\n", + "experiment.instrument.calib_sample_transparency = 0.24192 # FullProf SySin\n", "\n", - "experiment.peak.broad_gauss_u = 0.143361 # FullProf U\n", - "experiment.peak.broad_gauss_v = -0.522147 # FullProf V\n", - "experiment.peak.broad_gauss_w = 0.590413 # FullProf W\n", + "experiment.peak.broad_gauss_u = 0.143363 # FullProf U\n", + "experiment.peak.broad_gauss_v = -0.522167 # FullProf V\n", + "experiment.peak.broad_gauss_w = 0.590411 # FullProf W\n", "experiment.peak.broad_lorentz_x = 0.0 # FullProf X\n", - "experiment.peak.broad_lorentz_y = 0.054268 # FullProf Y\n", + "experiment.peak.broad_lorentz_y = 0.054276 # FullProf Y\n", "\n", "project.experiments.add(experiment)" ] @@ -177,19 +199,12 @@ "id": "11", "metadata": {}, "source": [ - "## Sample displacement / transparency (SyCos / SySin)\n", + "## Refine scale, sample displacement and transparency\n", "\n", - "FullProf applies sample-displacement (`SyCos`) and transparency\n", - "(`SySin`) peak-position shifts on top of `Zero` (see issue #117).\n", - "These map to `calib_sample_displacement` and\n", - "`calib_sample_transparency` on the CWL powder instrument. The cryspy\n", - "engine applies them only with the new functionality from\n", - "[cryspy PR #46](https://github.com/ikibalin/cryspy/pull/46); on the\n", - "currently released cryspy the corrections are ignored, and crysfml has\n", - "no equivalent, so this page stays in `ci_skip.txt` until a cryspy\n", - "release ships the support. As with `Zero` (`calib_twotheta_offset`\n", - "above), the cross-code convention may differ, so these `.pcr` values\n", - "may still need adjustment once validated against a PR #46 cryspy." + "FullProf's `SyCos`/`SySin` coefficients do not transfer numerically to\n", + "cryspy, and the two codes use a different absolute-intensity scale, so\n", + "`scale`, `calib_sample_displacement` and `calib_sample_transparency`\n", + "are refined against the FullProf profile using cryspy." ] }, { @@ -199,8 +214,11 @@ "metadata": {}, "outputs": [], "source": [ - "experiment.instrument.calib_sample_displacement = 0.05395 # FullProf SyCos\n", - "experiment.instrument.calib_sample_transparency = 0.09127 # FullProf SySin" + "experiment.calculator.type = 'cryspy'\n", + "experiment.linked_phases['lab6'].scale.free = True\n", + "experiment.instrument.calib_sample_displacement.free = True\n", + "experiment.instrument.calib_sample_transparency.free = True\n", + "project.analysis.fit()" ] }, { @@ -208,7 +226,11 @@ "id": "13", "metadata": {}, "source": [ - "## Calculate the pattern with each engine" + "## Calculate the pattern with each engine\n", + "\n", + "`cryspy` uses the refined `scale` and the refined sample-displacement\n", + "and transparency corrections. `crysfml` reuses the same refined `scale`\n", + "but cannot apply the corrections, so it keeps a peak-position offset." ] }, { @@ -229,10 +251,9 @@ "source": [ "## Compare each engine against FullProf\n", "\n", - "Until the corrections above are supported the engines will show\n", - "systematic peak-position offsets against the FullProf calculated\n", - "profile, which is the discrepancy this page is being prepared to\n", - "verify." + "After refinement `cryspy` reproduces the FullProf peak positions; the\n", + "residual is the remaining profile-shape difference. `crysfml` shows the\n", + "systematic peak-position offset expected without `SyCos`/`SySin`." ] }, { @@ -298,8 +319,11 @@ "source": [ "## Agreement check\n", "\n", - "Reported without failing CI (`raise_on_failure=False`) while the\n", - "corrections are unsupported; the page is also skipped via `ci_skip.txt`." + "Reported without failing CI (`raise_on_failure=False`): `cryspy`\n", + "reproduces the FullProf peak positions after refinement but a\n", + "profile-shape difference remains, and `crysfml` cannot apply the\n", + "corrections. The page is also skipped via `ci_skip.txt` until a\n", + "released cryspy ships the PR #46 support." ] }, { diff --git a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py index 83907b16a..a78bda9c7 100644 --- a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py +++ b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py @@ -1,11 +1,25 @@ # %% [markdown] # # LaB₆ — neutron powder, constant wavelength, Thompson–Cox–Hastings # -# A **prepared** verification for the FullProf `SyCos`/`SySin` systematic -# peak-position corrections (sample displacement and transparency), using -# the real LaB6 dataset from +# This page verifies the FullProf `SyCos`/`SySin` systematic +# peak-position corrections (sample displacement and transparency) for a +# constant-wavelength powder experiment, using the real LaB₆ dataset from # [cryspy issue #38](https://github.com/ikibalin/cryspy/issues/38). # +# `SyCos`/`SySin` map to `calib_sample_displacement` and +# `calib_sample_transparency` on the CWL powder instrument. Only the +# `cryspy` engine applies them, and only with the functionality added in +# [cryspy PR #46](https://github.com/ikibalin/cryspy/pull/46); the +# `crysfml` engine has no equivalent. Because FullProf and cryspy use +# **different coefficient conventions** for these corrections (and a +# different absolute-intensity scale), the `.pcr` values are used only as +# starting points: the page **refines** `scale`, `calib_sample_displacement` +# and `calib_sample_transparency` against the FullProf profile with cryspy, +# then reuses the refined `scale` for crysfml (which keeps a peak-position +# offset, since it cannot apply the corrections). +# +# The page stays listed in `ci_skip.txt` until a released cryspy ships the +# PR #46 corrections. # %% import easydiffraction as ed @@ -37,6 +51,10 @@ # %% [markdown] # ## Define the structure +# +# Boron is the ¹¹B isotope (FullProf `B11`). The `cryspy` engine resolves +# the isotope directly; the `crysfml` engine resolves scattering by +# element and silently drops the isotope number (`11B` → `B`). # %% structure = StructureFactory.from_scratch(name='lab6') @@ -49,14 +67,11 @@ fract_y=0.0, # FullProf Y fract_z=0.0, # FullProf Z adp_type='Biso', # FullProf Biso - adp_iso=0.53405, # FullProf Biso + adp_iso=0.53399, # FullProf Biso ) structure.atom_sites.create( label='B', # FullProf Atom - # ❌ returned a - # result with an exception set - # type_symbol='11B', # FullProf "B11 0.66500 0.00000 0" - type_symbol='B', # FullProf "B11 0.66500 0.00000 0" + type_symbol='11B', # FullProf "B11 0.66500 0.00000 0" fract_x=0.19972, # FullProf X fract_y=0.5, # FullProf Y fract_z=0.5, # FullProf Z @@ -68,6 +83,10 @@ # %% [markdown] # ## Create the experiment +# +# Starting values are taken from `ECH0030684_LaB6_1p622A.pcr`. The +# `SyCos`/`SySin` values are FullProf-convention starting points and are +# refined below into cryspy's equivalents. # %% experiment = ExperimentFactory.from_scratch( @@ -79,40 +98,42 @@ ) verify.set_reference_as_measured(experiment, x, calc_fullprof) -experiment.linked_phases.create(id='lab6', scale=136.0509) # FullProf Scale +experiment.linked_phases.create(id='lab6', scale=136.0507) # FullProf Scale experiment.instrument.setup_wavelength = 1.623891 # FullProf Lambda -experiment.instrument.calib_twotheta_offset = -0.45497 # FullProf Zero +experiment.instrument.calib_twotheta_offset = -0.45501 # FullProf Zero +experiment.instrument.calib_sample_displacement = 0.01052 # FullProf SyCos +experiment.instrument.calib_sample_transparency = 0.24192 # FullProf SySin -experiment.peak.broad_gauss_u = 0.143361 # FullProf U -experiment.peak.broad_gauss_v = -0.522147 # FullProf V -experiment.peak.broad_gauss_w = 0.590413 # FullProf W +experiment.peak.broad_gauss_u = 0.143363 # FullProf U +experiment.peak.broad_gauss_v = -0.522167 # FullProf V +experiment.peak.broad_gauss_w = 0.590411 # FullProf W experiment.peak.broad_lorentz_x = 0.0 # FullProf X -experiment.peak.broad_lorentz_y = 0.054268 # FullProf Y +experiment.peak.broad_lorentz_y = 0.054276 # FullProf Y project.experiments.add(experiment) # %% [markdown] -# ## Sample displacement / transparency (SyCos / SySin) +# ## Refine scale, sample displacement and transparency # -# FullProf applies sample-displacement (`SyCos`) and transparency -# (`SySin`) peak-position shifts on top of `Zero` (see issue #117). -# These map to `calib_sample_displacement` and -# `calib_sample_transparency` on the CWL powder instrument. The cryspy -# engine applies them only with the new functionality from -# [cryspy PR #46](https://github.com/ikibalin/cryspy/pull/46); on the -# currently released cryspy the corrections are ignored, and crysfml has -# no equivalent, so this page stays in `ci_skip.txt` until a cryspy -# release ships the support. As with `Zero` (`calib_twotheta_offset` -# above), the cross-code convention may differ, so these `.pcr` values -# may still need adjustment once validated against a PR #46 cryspy. +# FullProf's `SyCos`/`SySin` coefficients do not transfer numerically to +# cryspy, and the two codes use a different absolute-intensity scale, so +# `scale`, `calib_sample_displacement` and `calib_sample_transparency` +# are refined against the FullProf profile using cryspy. # %% -experiment.instrument.calib_sample_displacement = 0.05395 # FullProf SyCos -experiment.instrument.calib_sample_transparency = 0.09127 # FullProf SySin +experiment.calculator.type = 'cryspy' +experiment.linked_phases['lab6'].scale.free = True +experiment.instrument.calib_sample_displacement.free = True +experiment.instrument.calib_sample_transparency.free = True +project.analysis.fit() # %% [markdown] # ## Calculate the pattern with each engine +# +# `cryspy` uses the refined `scale` and the refined sample-displacement +# and transparency corrections. `crysfml` reuses the same refined `scale` +# but cannot apply the corrections, so it keeps a peak-position offset. # %% calc_ed_cryspy = verify.calculate_pattern(project, experiment, 'cryspy') @@ -121,10 +142,9 @@ # %% [markdown] # ## Compare each engine against FullProf # -# Until the corrections above are supported the engines will show -# systematic peak-position offsets against the FullProf calculated -# profile, which is the discrepancy this page is being prepared to -# verify. +# After refinement `cryspy` reproduces the FullProf peak positions; the +# residual is the remaining profile-shape difference. `crysfml` shows the +# systematic peak-position offset expected without `SyCos`/`SySin`. # %% project.display.pattern_comparison( @@ -159,8 +179,11 @@ # %% [markdown] # ## Agreement check # -# Reported without failing CI (`raise_on_failure=False`) while the -# corrections are unsupported; the page is also skipped via `ci_skip.txt`. +# Reported without failing CI (`raise_on_failure=False`): `cryspy` +# reproduces the FullProf peak positions after refinement but a +# profile-shape difference remains, and `crysfml` cannot apply the +# corrections. The page is also skipped via `ci_skip.txt` until a +# released cryspy ships the PR #46 support. # %% verify.assert_patterns_agree( From 22c6ea0d8398743019120005224601cae4e75163 Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Mon, 8 Jun 2026 20:30:25 +0200 Subject: [PATCH 10/41] Restructure LaB6 page: compare engines, then refine cryspy --- .../pd-neut-cwl_tch-fcj_lab6.ipynb | 205 ++++++++++++------ .../verification/pd-neut-cwl_tch-fcj_lab6.py | 127 +++++++---- 2 files changed, 222 insertions(+), 110 deletions(-) diff --git a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb index d315b5e3e..73cb53ec6 100644 --- a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb +++ b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb @@ -26,25 +26,26 @@ "source": [ "# LaB₆ — neutron powder, constant wavelength, Thompson–Cox–Hastings\n", "\n", - "This page verifies the FullProf `SyCos`/`SySin` systematic\n", - "peak-position corrections (sample displacement and transparency) for a\n", - "constant-wavelength powder experiment, using the real LaB₆ dataset from\n", + "This page calculates the **same** LaB₆ diffraction pattern with each\n", + "EasyDiffraction engine (`cryspy`, `crysfml`) and compares both against a\n", + "**FullProf** reference profile on identical input parameters, then\n", + "investigates the discrepancy by refinement. It uses the real LaB₆\n", + "dataset from\n", "[cryspy issue #38](https://github.com/ikibalin/cryspy/issues/38).\n", "\n", - "`SyCos`/`SySin` map to `calib_sample_displacement` and\n", - "`calib_sample_transparency` on the CWL powder instrument. Only the\n", - "`cryspy` engine applies them, and only with the functionality added in\n", + "The page targets the FullProf `SyCos`/`SySin` systematic peak-position\n", + "corrections (sample displacement and transparency), which map to\n", + "`calib_sample_displacement` and `calib_sample_transparency` on the CWL\n", + "powder instrument. Only the `cryspy` engine applies them, and only with\n", + "the functionality added in\n", "[cryspy PR #46](https://github.com/ikibalin/cryspy/pull/46); the\n", "`crysfml` engine has no equivalent. Because FullProf and cryspy use\n", - "**different coefficient conventions** for these corrections (and a\n", - "different absolute-intensity scale), the `.pcr` values are used only as\n", - "starting points: the page **refines** `scale`, `calib_sample_displacement`\n", - "and `calib_sample_transparency` against the FullProf profile with cryspy,\n", - "then reuses the refined `scale` for crysfml (which keeps a peak-position\n", - "offset, since it cannot apply the corrections).\n", - "\n", - "The page stays listed in `ci_skip.txt` until a released cryspy ships the\n", - "PR #46 corrections." + "different coefficient conventions for these corrections (and a\n", + "different absolute-intensity scale), the FullProf values are used as\n", + "starting points and the discrepancy is investigated by refining\n", + "`scale`, `calib_sample_displacement` and `calib_sample_transparency`\n", + "with `cryspy`. The page stays listed in `ci_skip.txt` until a released\n", + "cryspy ships the PR #46 corrections." ] }, { @@ -157,9 +158,7 @@ "source": [ "## Create the experiment\n", "\n", - "Starting values are taken from `ECH0030684_LaB6_1p622A.pcr`. The\n", - "`SyCos`/`SySin` values are FullProf-convention starting points and are\n", - "refined below into cryspy's equivalents." + "Starting values are taken from `ECH0030684_LaB6_1p622A.pcr`." ] }, { @@ -199,12 +198,7 @@ "id": "11", "metadata": {}, "source": [ - "## Refine scale, sample displacement and transparency\n", - "\n", - "FullProf's `SyCos`/`SySin` coefficients do not transfer numerically to\n", - "cryspy, and the two codes use a different absolute-intensity scale, so\n", - "`scale`, `calib_sample_displacement` and `calib_sample_transparency`\n", - "are refined against the FullProf profile using cryspy." + "## Calculate the pattern with each engine" ] }, { @@ -213,32 +207,6 @@ "id": "12", "metadata": {}, "outputs": [], - "source": [ - "experiment.calculator.type = 'cryspy'\n", - "experiment.linked_phases['lab6'].scale.free = True\n", - "experiment.instrument.calib_sample_displacement.free = True\n", - "experiment.instrument.calib_sample_transparency.free = True\n", - "project.analysis.fit()" - ] - }, - { - "cell_type": "markdown", - "id": "13", - "metadata": {}, - "source": [ - "## Calculate the pattern with each engine\n", - "\n", - "`cryspy` uses the refined `scale` and the refined sample-displacement\n", - "and transparency corrections. `crysfml` reuses the same refined `scale`\n", - "but cannot apply the corrections, so it keeps a peak-position offset." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "14", - "metadata": {}, - "outputs": [], "source": [ "calc_ed_cryspy = verify.calculate_pattern(project, experiment, 'cryspy')\n", "calc_ed_crysfml = verify.calculate_pattern(project, experiment, 'crysfml')" @@ -246,20 +214,21 @@ }, { "cell_type": "markdown", - "id": "15", + "id": "13", "metadata": {}, "source": [ "## Compare each engine against FullProf\n", "\n", - "After refinement `cryspy` reproduces the FullProf peak positions; the\n", - "residual is the remaining profile-shape difference. `crysfml` shows the\n", - "systematic peak-position offset expected without `SyCos`/`SySin`." + "At the FullProf input values the engines differ from the FullProf\n", + "reference: the absolute-intensity scale and the `SyCos`/`SySin`\n", + "coefficient conventions are not shared between codes, and `crysfml`\n", + "cannot apply the corrections at all." ] }, { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -275,7 +244,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -290,7 +259,7 @@ }, { "cell_type": "markdown", - "id": "18", + "id": "16", "metadata": {}, "source": [ "## Compare the two engines with each other" @@ -299,7 +268,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -314,22 +283,20 @@ }, { "cell_type": "markdown", - "id": "20", + "id": "18", "metadata": {}, "source": [ "## Agreement check\n", "\n", - "Reported without failing CI (`raise_on_failure=False`): `cryspy`\n", - "reproduces the FullProf peak positions after refinement but a\n", - "profile-shape difference remains, and `crysfml` cannot apply the\n", - "corrections. The page is also skipped via `ci_skip.txt` until a\n", - "released cryspy ships the PR #46 support." + "Reported without failing CI (`raise_on_failure=False`); the page is also\n", + "skipped via `ci_skip.txt` while the corrections require an unreleased\n", + "cryspy." ] }, { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -342,6 +309,114 @@ " raise_on_failure=False,\n", ")" ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "## Investigate the discrepancy by refinement\n", + "\n", + "Because the FullProf profile is already loaded as the measured data, we\n", + "can test directly whether `cryspy` can reproduce it: refine the\n", + "absolute `scale` together with `calib_sample_displacement` and\n", + "`calib_sample_transparency`, keeping the structure and every other\n", + "parameter fixed, and check how far the corrections have to move to\n", + "match FullProf. `crysfml` is not refined — it has no `SyCos`/`SySin`\n", + "equivalent." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "experiment.calculator.type = 'cryspy'\n", + "project.analysis.minimizer.type = 'lmfit'\n", + "\n", + "# Free only the absolute scale and the two peak-position corrections; the\n", + "# structure stays fixed, so a good fit confirms the difference is a\n", + "# scale/correction-convention difference, not a structural disagreement.\n", + "experiment.linked_phases['lab6'].scale.free = True\n", + "experiment.instrument.calib_sample_displacement.free = True\n", + "experiment.instrument.calib_sample_transparency.free = True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "project.analysis.fit()" + ] + }, + { + "cell_type": "markdown", + "id": "23", + "metadata": {}, + "source": [ + "## Goodness of fit and refined parameters\n", + "\n", + "The reference is a calculation-only profile with unit uncertainties, so\n", + "the absolute reduced χ² and R-factors are not normalised goodness-of-fit\n", + "values; the **scale-independent before/after closeness table** below is\n", + "the meaningful measure of the improvement." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "project.display.fit.results()" + ] + }, + { + "cell_type": "markdown", + "id": "25", + "metadata": {}, + "source": [ + "## Refined cryspy vs FullProf\n", + "\n", + "The refined `cryspy` pattern overlaid on the FullProf reference, then a\n", + "before/after table of the closeness metrics. A residual profile-shape\n", + "difference remains, which is why the page is not yet a passing\n", + "regression check." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26", + "metadata": {}, + "outputs": [], + "source": [ + "calc_ed_cryspy_refined = verify.calculate_pattern(project, experiment, 'cryspy')\n", + "\n", + "project.display.pattern_comparison(\n", + " 'lab6',\n", + " reference=calc_fullprof,\n", + " candidate=calc_ed_cryspy_refined,\n", + " reference_label='FullProf',\n", + " candidate_label='EasyDiffraction (cryspy, refined)',\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "verify.report_refinement_closeness(calc_fullprof, calc_ed_cryspy, calc_ed_cryspy_refined)" + ] } ], "metadata": { diff --git a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py index a78bda9c7..4aeff459f 100644 --- a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py +++ b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py @@ -1,25 +1,26 @@ # %% [markdown] # # LaB₆ — neutron powder, constant wavelength, Thompson–Cox–Hastings # -# This page verifies the FullProf `SyCos`/`SySin` systematic -# peak-position corrections (sample displacement and transparency) for a -# constant-wavelength powder experiment, using the real LaB₆ dataset from +# This page calculates the **same** LaB₆ diffraction pattern with each +# EasyDiffraction engine (`cryspy`, `crysfml`) and compares both against a +# **FullProf** reference profile on identical input parameters, then +# investigates the discrepancy by refinement. It uses the real LaB₆ +# dataset from # [cryspy issue #38](https://github.com/ikibalin/cryspy/issues/38). # -# `SyCos`/`SySin` map to `calib_sample_displacement` and -# `calib_sample_transparency` on the CWL powder instrument. Only the -# `cryspy` engine applies them, and only with the functionality added in +# The page targets the FullProf `SyCos`/`SySin` systematic peak-position +# corrections (sample displacement and transparency), which map to +# `calib_sample_displacement` and `calib_sample_transparency` on the CWL +# powder instrument. Only the `cryspy` engine applies them, and only with +# the functionality added in # [cryspy PR #46](https://github.com/ikibalin/cryspy/pull/46); the # `crysfml` engine has no equivalent. Because FullProf and cryspy use -# **different coefficient conventions** for these corrections (and a -# different absolute-intensity scale), the `.pcr` values are used only as -# starting points: the page **refines** `scale`, `calib_sample_displacement` -# and `calib_sample_transparency` against the FullProf profile with cryspy, -# then reuses the refined `scale` for crysfml (which keeps a peak-position -# offset, since it cannot apply the corrections). -# -# The page stays listed in `ci_skip.txt` until a released cryspy ships the -# PR #46 corrections. +# different coefficient conventions for these corrections (and a +# different absolute-intensity scale), the FullProf values are used as +# starting points and the discrepancy is investigated by refining +# `scale`, `calib_sample_displacement` and `calib_sample_transparency` +# with `cryspy`. The page stays listed in `ci_skip.txt` until a released +# cryspy ships the PR #46 corrections. # %% import easydiffraction as ed @@ -84,9 +85,7 @@ # %% [markdown] # ## Create the experiment # -# Starting values are taken from `ECH0030684_LaB6_1p622A.pcr`. The -# `SyCos`/`SySin` values are FullProf-convention starting points and are -# refined below into cryspy's equivalents. +# Starting values are taken from `ECH0030684_LaB6_1p622A.pcr`. # %% experiment = ExperimentFactory.from_scratch( @@ -113,27 +112,8 @@ project.experiments.add(experiment) -# %% [markdown] -# ## Refine scale, sample displacement and transparency -# -# FullProf's `SyCos`/`SySin` coefficients do not transfer numerically to -# cryspy, and the two codes use a different absolute-intensity scale, so -# `scale`, `calib_sample_displacement` and `calib_sample_transparency` -# are refined against the FullProf profile using cryspy. - -# %% -experiment.calculator.type = 'cryspy' -experiment.linked_phases['lab6'].scale.free = True -experiment.instrument.calib_sample_displacement.free = True -experiment.instrument.calib_sample_transparency.free = True -project.analysis.fit() - # %% [markdown] # ## Calculate the pattern with each engine -# -# `cryspy` uses the refined `scale` and the refined sample-displacement -# and transparency corrections. `crysfml` reuses the same refined `scale` -# but cannot apply the corrections, so it keeps a peak-position offset. # %% calc_ed_cryspy = verify.calculate_pattern(project, experiment, 'cryspy') @@ -142,9 +122,10 @@ # %% [markdown] # ## Compare each engine against FullProf # -# After refinement `cryspy` reproduces the FullProf peak positions; the -# residual is the remaining profile-shape difference. `crysfml` shows the -# systematic peak-position offset expected without `SyCos`/`SySin`. +# At the FullProf input values the engines differ from the FullProf +# reference: the absolute-intensity scale and the `SyCos`/`SySin` +# coefficient conventions are not shared between codes, and `crysfml` +# cannot apply the corrections at all. # %% project.display.pattern_comparison( @@ -179,11 +160,9 @@ # %% [markdown] # ## Agreement check # -# Reported without failing CI (`raise_on_failure=False`): `cryspy` -# reproduces the FullProf peak positions after refinement but a -# profile-shape difference remains, and `crysfml` cannot apply the -# corrections. The page is also skipped via `ci_skip.txt` until a -# released cryspy ships the PR #46 support. +# Reported without failing CI (`raise_on_failure=False`); the page is also +# skipped via `ci_skip.txt` while the corrections require an unreleased +# cryspy. # %% verify.assert_patterns_agree( @@ -194,3 +173,61 @@ ], raise_on_failure=False, ) + +# %% [markdown] +# ## Investigate the discrepancy by refinement +# +# Because the FullProf profile is already loaded as the measured data, we +# can test directly whether `cryspy` can reproduce it: refine the +# absolute `scale` together with `calib_sample_displacement` and +# `calib_sample_transparency`, keeping the structure and every other +# parameter fixed, and check how far the corrections have to move to +# match FullProf. `crysfml` is not refined — it has no `SyCos`/`SySin` +# equivalent. + +# %% +experiment.calculator.type = 'cryspy' +project.analysis.minimizer.type = 'lmfit' + +# Free only the absolute scale and the two peak-position corrections; the +# structure stays fixed, so a good fit confirms the difference is a +# scale/correction-convention difference, not a structural disagreement. +experiment.linked_phases['lab6'].scale.free = True +experiment.instrument.calib_sample_displacement.free = True +experiment.instrument.calib_sample_transparency.free = True + +# %% +project.analysis.fit() + +# %% [markdown] +# ## Goodness of fit and refined parameters +# +# The reference is a calculation-only profile with unit uncertainties, so +# the absolute reduced χ² and R-factors are not normalised goodness-of-fit +# values; the **scale-independent before/after closeness table** below is +# the meaningful measure of the improvement. + +# %% +project.display.fit.results() + +# %% [markdown] +# ## Refined cryspy vs FullProf +# +# The refined `cryspy` pattern overlaid on the FullProf reference, then a +# before/after table of the closeness metrics. A residual profile-shape +# difference remains, which is why the page is not yet a passing +# regression check. + +# %% +calc_ed_cryspy_refined = verify.calculate_pattern(project, experiment, 'cryspy') + +project.display.pattern_comparison( + 'lab6', + reference=calc_fullprof, + candidate=calc_ed_cryspy_refined, + reference_label='FullProf', + candidate_label='EasyDiffraction (cryspy, refined)', +) + +# %% +verify.report_refinement_closeness(calc_fullprof, calc_ed_cryspy, calc_ed_cryspy_refined) From 8570ce6e36ceb7a551f7c23c658557339d6b318b Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Mon, 8 Jun 2026 20:33:49 +0200 Subject: [PATCH 11/41] Drop Zero offset from LaB6 page to match other pages --- docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb | 1 - docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py | 1 - 2 files changed, 2 deletions(-) diff --git a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb index 73cb53ec6..cb080bfce 100644 --- a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb +++ b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.ipynb @@ -180,7 +180,6 @@ "experiment.linked_phases.create(id='lab6', scale=136.0507) # FullProf Scale\n", "\n", "experiment.instrument.setup_wavelength = 1.623891 # FullProf Lambda\n", - "experiment.instrument.calib_twotheta_offset = -0.45501 # FullProf Zero\n", "experiment.instrument.calib_sample_displacement = 0.01052 # FullProf SyCos\n", "experiment.instrument.calib_sample_transparency = 0.24192 # FullProf SySin\n", "\n", diff --git a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py index 4aeff459f..23e43fd1d 100644 --- a/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py +++ b/docs/docs/verification/pd-neut-cwl_tch-fcj_lab6.py @@ -100,7 +100,6 @@ experiment.linked_phases.create(id='lab6', scale=136.0507) # FullProf Scale experiment.instrument.setup_wavelength = 1.623891 # FullProf Lambda -experiment.instrument.calib_twotheta_offset = -0.45501 # FullProf Zero experiment.instrument.calib_sample_displacement = 0.01052 # FullProf SyCos experiment.instrument.calib_sample_transparency = 0.24192 # FullProf SySin From 570a8e4b1d8c8aab5c2174ae5d877b91a1f35c7e Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Tue, 9 Jun 2026 14:24:48 +0200 Subject: [PATCH 12/41] Add public Analysis.calculate() method Mirrors fit() as the non-fitting trigger that refreshes structures and experiments so experiment.data.intensity_calc reflects current params. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/easydiffraction/analysis/analysis.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/easydiffraction/analysis/analysis.py b/src/easydiffraction/analysis/analysis.py index e16d55c83..6d82dda76 100644 --- a/src/easydiffraction/analysis/analysis.py +++ b/src/easydiffraction/analysis/analysis.py @@ -1287,6 +1287,22 @@ def fit( except KeyboardInterrupt: self._handle_fit_interrupted(verbosity=verb) + def calculate(self) -> None: + """ + Calculate the diffraction pattern for every experiment. + + Refreshes the linked structures and each experiment so the + calculated intensities (``experiment.data.intensity_calc``) + reflect the current parameters and the selected calculation + engines. This is the non-fitting counterpart of :meth:`fit`: call + it after changing parameters or a calculator to update the + calculated pattern without running a minimization. + """ + for structure in self.project.structures: + structure._update_categories() + for experiment in self.project.experiments: + experiment._update_categories() + def undo_fit(self) -> UndoFitOutcome: """ Roll back the latest fit output and scalar state. From 52fbf01bb5bd144c671c5ea920a5015c14a27eee Mon Sep 17 00:00:00 2001 From: Andrew Sazonov Date: Tue, 9 Jun 2026 14:24:48 +0200 Subject: [PATCH 13/41] Refactor FullProf verification loaders for project-dir API Loaders take (project_dir, file) and hide bundled_reference_dir; add load_fullprof_calc_profile for .prf Icalc minus real .bac background, handling both .bac layouts (two-column and header+flat-array). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/easydiffraction/analysis/verification.py | 223 ++++++++++++++++++- 1 file changed, 220 insertions(+), 3 deletions(-) diff --git a/src/easydiffraction/analysis/verification.py b/src/easydiffraction/analysis/verification.py index b73c73222..5235121c7 100644 --- a/src/easydiffraction/analysis/verification.py +++ b/src/easydiffraction/analysis/verification.py @@ -32,6 +32,19 @@ # columns: h, k, l, ivk, cod, F2obs, F2cal. _MIN_SC_F2CAL_COLUMNS = 7 +# A FullProf ``.prf`` profile data row (Prf=-3 format) has exactly these +# columns: 2Theta, Yobs, Ycal, Yobs-Ycal, Backg. Reflection-marker rows +# carry a trailing ``(h k l)`` and more columns, so they are skipped. The +# calculated intensity is column 2 (Ycal). +_PRF_PROFILE_COLUMNS = 5 +_PRF_YCALC_COLUMN = 2 + +# A FullProf ``Prf=2`` IGOR profile row has columns TwoTheta, Iobs, +# Icalc, Diff under a ``BEGIN``/``END`` block; the calculated intensity +# is column 2 (Icalc). +_IGOR_ICALC_COLUMN = 2 +_IGOR_MIN_COLUMNS = 3 + # FullProf writes a profile header (min, increment, max) in three fixed # columns of this width; adjacent values run together when one fills its # field, so the header is sliced by column when it cannot be split on @@ -102,10 +115,13 @@ def _parse_fullprof_header(line: str) -> tuple[float, float, float]: return minimum, increment, maximum -def load_fullprof_profile(path: str) -> tuple[np.ndarray, np.ndarray]: +def load_fullprof_profile(project_dir: str, profile_file: str) -> tuple[np.ndarray, np.ndarray]: """ Load a FullProf ``.sub``/``.sim`` profile as ``(x, y)`` arrays. + Resolved inside the bundled reference directory, so the caller passes + the project sub-folder and the file name. + The first line holds ``min increment max`` followed by a comment; the x grid is reconstructed from that header and the remaining lines are flattened into the intensity array. The header is parsed by @@ -130,6 +146,7 @@ def load_fullprof_profile(path: str) -> tuple[np.ndarray, np.ndarray]: from the grid implied by the intensities read from the body (a genuine inconsistency rather than header rounding). """ + path = str(bundled_reference_dir() / project_dir / profile_file) with Path(path).open(encoding='utf-8') as handle: lines = handle.readlines() if not lines: @@ -158,7 +175,8 @@ def load_fullprof_profile(path: str) -> tuple[np.ndarray, np.ndarray]: def load_columned_profile( - path: str, + project_dir: str, + profile_file: str, *, skip_rows: int = 1, columns: tuple[int, int] = (0, 1), @@ -183,6 +201,7 @@ def load_columned_profile( tuple[np.ndarray, np.ndarray] The x and y arrays. """ + path = str(bundled_reference_dir() / project_dir / profile_file) with Path(path).open(encoding='utf-8') as handle: lines = handle.readlines()[skip_rows:] cleaned = '\n'.join(line.replace('(', ' ').replace(')', ' ') for line in lines) @@ -190,10 +209,207 @@ def load_columned_profile( return x, y -def load_fullprof_sc_f2calc(path: str) -> dict[tuple[int, int, int], float]: +def _parse_fullprof_calc_profile(path: str) -> tuple[np.ndarray, np.ndarray]: + """ + Read ``(2θ, Icalc)`` from a FullProf calculated-profile export. + + Handles both the ``Prf=2`` IGOR text format (``TwoTheta Iobs Icalc + Diff`` rows inside a ``BEGIN``/``END`` block) and the tab-separated + ``Prf=-3`` format (``2Theta Yobs Ycal …`` under a ``2Theta``-led + header, with interleaved reflection-marker rows skipped). Both place + the calculated intensity in column 2. + + Parameters + ---------- + path : str + Path to the FullProf ``.prf`` file. + + Returns + ------- + tuple[np.ndarray, np.ndarray] + The 2θ grid (corrected axis) and the calculated intensities. + + Raises + ------ + ValueError + If no profile data rows are found. + """ + lines = Path(path).read_text(encoding='utf-8').splitlines() + two_theta: list[float] = [] + icalc: list[float] = [] + is_igor = any('IGOR' in line.upper() for line in lines[:3]) + if is_igor: + started = False + for line in lines: + token = line.strip() + if token == 'BEGIN': + started = True + continue + if token == 'END': + break + fields = token.split() + if not started or len(fields) < _IGOR_MIN_COLUMNS: + continue + try: + values = (float(fields[0]), float(fields[_IGOR_ICALC_COLUMN])) + except ValueError: + continue + two_theta.append(values[0]) + icalc.append(values[1]) + else: + header_index = next( + (index for index, line in enumerate(lines) if line.lstrip().startswith('2Theta')), + None, + ) + if header_index is None: + msg = f'FullProf profile {path}: no "2Theta" header or IGOR block found.' + raise ValueError(msg) + for line in lines[header_index + 1 :]: + if '(' in line: # reflection-marker row + continue + fields = line.split() + if len(fields) != _PRF_PROFILE_COLUMNS: + continue + two_theta.append(float(fields[0])) + icalc.append(float(fields[_PRF_YCALC_COLUMN])) + if not two_theta: + msg = f'FullProf profile {path}: no calculated-profile data rows found.' + raise ValueError(msg) + return np.asarray(two_theta), np.asarray(icalc) + + +def _parse_fullprof_background(path: str) -> tuple[np.ndarray, np.ndarray]: + """ + Read ``(2θ, background)`` from a FullProf ``Ppl=2`` ``.bac`` file. + + FullProf writes the ``.bac`` in one of two layouts, both handled here: + + * **Two-column** — a ``!``-prefixed comment line followed by + ``2Theta background`` rows. + * **Header + array** — a ``min step max`` header line (``.sub``-style, + with a trailing ``Background of: …`` label) followed by the + background values as a flat array (several per row); the 2θ grid is + reconstructed from the header. + + In both layouts the 2θ axis omits the zero shift, so the caller + realigns it onto the profile axis. + + Parameters + ---------- + path : str + Path to the FullProf ``.bac`` file. + + Returns + ------- + tuple[np.ndarray, np.ndarray] + The (uncorrected) 2θ grid and the real background intensities. + + Raises + ------ + ValueError + If no background data rows are found. + """ + lines = [line for line in Path(path).read_text(encoding='utf-8').splitlines() if line.strip()] + if not lines: + msg = f'FullProf .bac {path}: file is empty.' + raise ValueError(msg) + + if not lines[0].lstrip().startswith('!'): + # Header + flat-array layout: first line is ``min step max