diff --git a/astro_data/corrections/README.md b/astro_data/corrections/README.md new file mode 100644 index 000000000..b8481c074 --- /dev/null +++ b/astro_data/corrections/README.md @@ -0,0 +1,172 @@ +# Catalog corrections + +A **correction** is a cited value that replaces what a source gives, because +the source is wrong for one object. The catalog import applies every +correction in this directory as its final pass, so a correction always has +the last word. It fails the import rather than apply one that no longer fits +the data. + +Corrections are the fix of last resort. Before writing one, check that the +fix belongs here and not in one of the other three places +([`docs/ax/catalog.md`](../../docs/ax/catalog.md) §9.2, and +[ADR 0046](../../docs/adr/0046-catalog-corrections-cited-checked-applied-last.md)): + +| the cause | the fix | +|---|---| +| a loader misreads its source | fix the loader | +| one better reference covers more than a handful of objects by a stable key | a supplementary source | +| a curated source (EGC, `bright_stars.csv`, `messier_names.dat`) is wrong | edit that file | +| a published source is wrong for one object | **a correction, here** | + +If you are about to write more than a handful of corrections citing the same +reference, it is a supplementary source. + +## Files + +One TOML file per published source, named after it: `steinicke.toml`, +`sac_multistars.toml`, `ngc2000_names.toml`. Each file is also the list of +errors to report to that source's publisher. + +**A correction goes in the file of the source whose error it fixes, even when +its key is another catalog's.** NGC 2000.0's names file puts "Horsehead +nebula" on IC 434. Both halves of the fix go in `ngc2000_names.toml`: the +removal keyed `["IC 434"]`, and the name keyed `["B 33"]`. + +Files are read in name order, and tables in file order. Other files here, +such as this README, are ignored. + +## Tables + +Each correction is one table, headed by a **designator**: a catalog code, one +space, and the number, quoted. The code is the one in the database's +`catalogs` table, such as `NGC`, `IC`, `M`, `Sh2`, `SaM` or `WDS`. + +```toml +["IC 424"] +why = "Steinicke gives 120'; the nebula is 2.5' across." +sources = ["https://…"] +size = "2.5'" +was.size = "120'" +``` + +The table's correction applies to the **sky object** that the designator +resolves to, which every catalog listing on it shares, with one exception: +a `description` belongs to that one catalog listing. `["M 42"]` and +`["NGC 1976"]` reach the same sky object, so both may not correct its +magnitude. + +Every table needs: + +- **`why`**: what is wrong, and what the sources say. +- **`sources`**: a list of citations: a URL where one exists, otherwise a + full citation (a book and page, or a bibcode). + +## Fields + +A table can set any of these fields, named like the database columns. Each +field needs **`was.`**, the value it replaces, written the same way. +If the field is empty in the database, write `was. = ""`. + +| field | written as | example | +|---|---|---| +| `obj_type` | a type code from `PiFinder/obj_types.py` | `"C+N"` | +| `ra` | sexagesimal hours | `"05 35 17.3"`, `"05:35:17.3"`, `"5h35m17.3s"` | +| `dec` | sexagesimal degrees, signed | `"-05 23 28"`, `"+41°16′09″"`, `"-00 30 00"` | +| `const` | an IAU abbreviation | `"Eri"` | +| `size` | arcminutes, or with a unit: `'`, `"`, `°` | `"2.5'"`, `"60'x30'"`, `"36\""` | +| `mag` | a number, or a list for a variable's range | `11.4`, `[4.86, 7.32]` | +| `description` | text; a multi-line string is fine | `"""…"""` | + +- `ra` and `dec` take two or three parts, and only the last may have + decimals. A bare number is refused, because it could be hours or degrees. +- `size` is read by the loaders' own `parse_arcmin_size`, and must parse + whole. It corrects the extents. The position angle stays as it was. +- Correcting `ra` or `dec` recomputes the constellation from the new + position, as a loader would. To overrule that, for a boundary case like + NGC 1404, correct `const` too. + +## Names + +```toml +["M 82"] +why = "NGC 2000.0 gives the M 81/M 82 pair's name to M 82 alone." +sources = ["https://…"] +names.remove = ["Bode's nebulae"] +names.preferred = "Cigar Galaxy" +``` + +- **`names.add`**: a list of names to append. +- **`names.remove`**: a list of names to delete. Every copy goes. +- **`names.preferred`**: one name, placed first among the sky object's names + so the device and the atlas show it. It is added if the sky object lacks + it. + +Names are compared exactly, including case. + +## Identifications + +These correct which sky object a designator belongs to. + +- **`same_as = "IC 2480"`**: this designator is the same sky object as that + one. Its listing moves there. If it was the sky object's only listing, the + whole sky object merges in, names and all, and is deleted. The target's + values stand and the merged object's are dropped, so correct anything + worth keeping in the same table. +- **`own_object = true`**: this designator is a sky object of its own. Its + listing moves to a new sky object that starts as a copy of the old one. + Correct its fields in the same table, with `was` stating the copied + values. +- **`hip = 102989`**: this star listing is that Hipparcos star. Use it where + the source's position is too far off for the star cross-match to find it. + +A listing moves with the names from its own catalog that are its +designator: its own "IC 5281", or any form the loaders' lookup resolves to +it, such as "Collinder 42" from Col. Its other names stay where they are. +If it has no such name, as with an M listing, whose "M nn" name comes +with Steinicke's NGC record, the designator is added as a name where it +lands. + +All splits (`own_object`) run before all joins (`same_as`), and both run +before any other correction. So the other fields in an identification's +table, and `was`, apply to the sky object the designator ends up on. + +`hip` is the one correction the final pass does not apply. The star +cross-match reads it before it matches, and uses it in place of the +position. + +## Checks + +The import fails, naming the file and the designator, when: + +- a file isn't valid TOML, or a table has an unknown key or a malformed + value; +- `why` or `sources` is missing; +- a field has no `was.`, or `was` equals the correction; +- a field already holds the corrected value (**redundant**: delete the + correction); +- a field holds neither `was` nor the correction (**stale**: the data + underneath has changed, so check the correction against its source); +- a designator resolves to nothing; +- two corrections reach the same field, or the same name, of one sky object; +- two corrections identify the same listing, or both give it a `hip`; +- `names.remove` names a name the sky object doesn't have, `names.add` one it + already has, or `names.preferred` the name that is already first; +- a `same_as` is already true, or an `own_object` designator is already its + own sky object; +- a `same_as` points at a designator that has its own `same_as` (point it at + the final sky object instead); +- a split or join leaves its old sky object still carrying another + catalog's name that the loaders' lookup resolves to the designator: its + exact key ("NGC70"), or a designation `parse_designation` reads as it + ("Cr 42" for Col 42, "NGC 0070" for NGC 70, but not "H 24", since a bare + "H" names no catalog). That catalog's listing was matched through the + wrong identification and would be left behind, so correct it too. If the + name is only a stray, a `names.remove` on the old sky object clears it. + That is the usual case for an M listing: Steinicke's NGC record carries + its "M nn" name, so splitting or moving one needs a `names.remove` of that + name keyed by the NGC designator, which records that the NGC record's + cross-identification is wrong. + +A check compares values within tolerances: 0.05 for magnitudes, 1″ for RA +and Dec, each taken separately, and 1″ for each size extent once parsed. So +`"120'"` and `"2°"` are the same size. diff --git a/docs/adr/0046-catalog-corrections-cited-checked-applied-last.md b/docs/adr/0046-catalog-corrections-cited-checked-applied-last.md new file mode 100644 index 000000000..e787d7207 --- /dev/null +++ b/docs/adr/0046-catalog-corrections-cited-checked-applied-last.md @@ -0,0 +1,46 @@ +# Catalog errors are fixed in one of three places; source errors become cited, checked corrections applied last + +**Status: accepted** (2026-10-22). Vocabulary: [`docs/ax/catalog/CONTEXT.md`](../ax/catalog/CONTEXT.md), *Import*. Architecture: [`docs/ax/catalog.md`](../ax/catalog.md) §9. + +An audit of `objects.db` against OpenNGC and Hipparcos (paperChart_dev, 2026-10) found about 400 wrong values: types, names, positions, magnitudes, sizes. Most come from **loaders** misreading their **sources**. A substantial minority come from the sources themselves — Steinicke's 120′ for IC 424, SAC's "Beid" for Keid, NGC 2000.0's "Bode's nebulae" on M 82 — which a loader reads faithfully and must not "fix" by guessing. PiFinder had no place for a value that contradicts its source: the only precedent was `extra_names.dat`, three lines no loader reads. + +## Decision + +1. **Every wrong value is fixed in exactly one of three places.** + - The loader misreads its source → **fix the loader**. + - A published source is wrong, and one better reference covers more than a handful of objects by a stable key → **a supplementary source**: a vendored reference joined on that key (Hipparcos and IAU WGSN by HIP, Bica et al. 2008 by designation), which fills or replaces one field by rule and never adds listings. + - A published source is wrong for one object, and no rule or reference can produce the value → **a correction**. + + A **curated source** — a list PiFinder compiles itself, such as EGC or the bright stars — is none of these: it is edited directly. + +2. **A correction is keyed by a designator** (`"NGC 604"`) and applies to the sky object that designator resolves to; a description correction applies to that one catalog listing. `objects.id` is an autoincrement and changes on every rebuild, so it cannot be the key. Two corrections that reach the same field of one sky object through different designators (M 42, NGC 1976) fail the build, as does a designator that resolves to nothing. + +3. **A correction does one of three things:** set fields (`obj_type`, `ra`, `dec`, `const`, `size`, `mag`, `description`); edit names (`names.add`, `names.remove`, and `names.preferred`, which puts a name first); or correct an **identification** — in either direction (`same_as = "IC 2480"`, or `own_object = true` to detach a designator into a sky object of its own), or, for a star listing, by naming its Hipparcos number (`hip = 102989`). It never creates a sky object from nothing or deletes a listing: every orphan and duplicate the audit found is a loader bug. + +4. **Corrections are applied in one final pass**, after every loader, the star cross-match, the supplementary sources and type normalisation, and before image resolution and the database is finalised. A correction therefore always has the last word, and its type is checked against the normalised type codes. A split (`own_object`) or a join (`same_as`) fails the build if the sky object it leaves still carries another catalog's name equal to the moved designator, since that catalog's listing was matched through the wrong identification and the move would leave it behind. + + **The one exception is `hip`.** Stars are identified by cross-matching each listing's position to Hipparcos, and a source position whole degrees off (SAC's α Mic and ξ Lup) finds no match — so a position corrected in the final pass would come too late to merge the star or to let Hipparcos supply its values. The star-identity stage therefore reads `hip` corrections before it cross-matches, and uses them in place of position. An identification feeds matching; every other correction overwrites a value, and those stay last. A `hip` correction is redundant if the cross-match alone finds that number. + +5. **The format is TOML**: one table per designator, one file per corrected source, under `astro_data/corrections/` (`steinicke.toml`, `sac_multistars.toml`, `ngc2000_names.toml`…). Field names are the database's columns; values are in the units references publish — sexagesimal RA and Dec, sizes through the loaders' own `parse_arcmin_size`, a magnitude as a number or a variable's range as a list. `why` and `sources` (URLs) are required. + +6. **Every field correction states the value it replaces** (`was.size = "120'"`), and the pass checks it, within tolerances (0.05 mag, 1″, sizes compared parsed). If the field already holds the corrected value, the correction is redundant; if it holds anything other than `was`, the correction is stale. Both fail the build. Name and identification operations carry the same check implicitly: a `names.remove` of an absent name, or a `same_as` on designators already one sky object, fails. + +## Considered options + +- **Edit the published source files.** Rejected: a vendored source would no longer be what its publisher released, an updated release could not be dropped in, and nothing would record what was changed or why. Curated sources are the exception because PiFinder is their publisher. +- **Special-case objects in loader code.** Rejected: values without citations, scattered across thirteen loaders, invisible to anyone reviewing the data. `add_missing_messier_objects` is the existing example of the pattern, and is left as it is rather than extended. +- **Corrections for the systematic errors too** — copy Bica's 49 Magellanic sizes and WGSN's names in by hand. Rejected: hundreds of transcribed rows that drift from their reference and cannot be regenerated. That is what a supplementary source is for. +- **Apply corrections at insert time, as each loader writes.** Rejected: a later loader's fill or a supplementary source's replacement would land on top, and a correction would no longer have the last word. +- **Build the whole catalog in memory, correct it, write once.** The cleanest reading of "correct before it reaches the DB", but it rewrites every loader. The database being built is a scratch file until it is finalised, so a final pass gives the same guarantee — the shipped file never holds an uncorrected value — for a fraction of the change. +- **Key by `object_id` or by position.** `object_id` is unstable across rebuilds; positions are ambiguous, since the audit found 582 groups of NGC/IC objects at identical coordinates. +- **YAML, TSV, JSON Lines.** YAML 1.1 reads `13:42:47` as a base-60 integer — a trap for RA. TSV cannot express name edits or identifications. JSON has no comments, and hand-editing it is error-prone. TOML has comments and multi-line strings, diffs cleanly, and matches paperChart's `labels.toml`. +- **Warn on stale or redundant corrections instead of failing.** Rejected: the import runs on a developer's machine, where a failure costs a minute, and a warning in a long import log is a warning nobody reads. + +## Consequences + +- **Corrections are written last, after the loader fixes.** Those fixes change what some designators resolve to and what values sit underneath — attaching Steinicke's duplicates to their twins moves `NGC 2239` onto `NGC 2244`'s sky object — and the `was` check reports exactly which corrections they disturbed. +- **Re-vendoring a source fails the import until its corrections are reviewed.** That is the point: a correction Steinicke has since fixed should be deleted, not silently re-applied over the fix. +- **Each corrections file is the upstream report for its publisher.** `sac_multistars.toml` is the list to send SAC. +- **Unresolved identifications stay as the sources give them.** A correction needs a decided value and a citation; NGC 6002, NGC 3355 and the other rows the audit could not settle have neither. +- **Name order carries meaning**, because `names.preferred` and the WGSN names work by placing a name first. The import must therefore keep names in a reproducible order; a `set()` of names follows Python's per-process hash seed and does not. +- **The import gains a development dependency**: `tomli` on Python 3.9, `tomllib` from 3.11. The device never runs the import. diff --git a/docs/ax/catalog.md b/docs/ax/catalog.md index 3090cf71d..1079d3334 100644 --- a/docs/ax/catalog.md +++ b/docs/ax/catalog.md @@ -4,7 +4,9 @@ This document describes how PiFinder loads, organizes, filters, searches, and updates astronomical catalogs at runtime. The bulk of the system lives in `PiFinder/catalogs.py`, with supporting types in `PiFinder/catalog_base.py` and `PiFinder/composite_object.py`, and the -on-disk SQLite layer in `PiFinder/db/objects_db.py`. +on-disk SQLite layer in `PiFinder/db/objects_db.py`. Section 9 covers +the offline **catalog import** in `PiFinder/catalog_imports/`, which +builds that SQLite file from the sources in `astro_data/`. For the canonical glossary of terms and data structures, see [`catalog/CONTEXT.md`](./catalog/CONTEXT.md). @@ -403,7 +405,220 @@ of logged status, for example). --- -## 9. Glossary +## 9. Building the database: the catalog import + +`astro_data/pifinder_objects.db` is a build artifact. `main()` in +`catalog_imports/main.py` drops every table and rebuilds the file from +the **sources** in `astro_data/`. The import runs on a developer's +machine; a device never runs it, and ships the committed file instead. + +> **Status (2026-10-22).** Parts of this section describe the import as +> the catalog-audit fixes leave it, not as `main` has it today: the +> match guard (9.3), the star cross-match (9.3) and the supplementary +> sources (9.4) are being built, and the designation allowlist (9.3) is +> being extracted from the NixOS branch (#379) to land on its own. The +> correction pass (9.5) is in `catalog_imports/corrections.py`, and +> `astro_data/corrections/` holds no corrections yet. +> The decision record is [ADR 0046](../adr/0046-catalog-corrections-cited-checked-applied-last.md). + +### 9.1 Pipeline + +``` + sources (astro_data/) + │ + ▼ + loaders, in CATALOG_LOADERS order (main.py) + │ each listing: parse its akas → match guard → + │ join an existing sky object, or mint a new one + ▼ + post-processing (POST_PROCESSING_FUNCTIONS) + │ type normalisation, the hand-built Messier objects + ▼ + star identity: Hipparcos cross-match, merge by HIP + │ (`hip` corrections are read here, ahead of position) + ▼ + supplementary sources: Hipparcos values → WGSN names → Bica sizes + ▼ + correction pass (astro_data/corrections/*.toml) + ▼ + resolve_object_images → finalise (WAL checkpoint, DELETE journal) +``` + +Each stage has the last word over the stages before it. Loader order +matters: a loader can only join sky objects that earlier loaders created +(so Sharpless and Collinder load before Caldwell, whose C 9 and C 41 +are Sh2 155 and Col 50). + +When a loader joins a sky object an earlier loader created, it may only +fill what is empty — never replace — with two named exceptions: + +| field | rule | +|---|---| +| magnitude, size | fill if the sky object has none (`filter_mag == 99`, no extents) | +| magnitude of a globular | **Harris replaces**: it is the reference for Milky Way globulars | +| type | the first loader's stands; **EGC sets `Gb`** — a cluster on PiFinder's globular list is a globular | +| position | the first loader's stands (for stars, Hipparcos replaces it later) | + +A new exception belongs in this table, not in a loader's `if`. + +### 9.2 Where a fix goes + +When the database holds a wrong value, the fix goes in exactly one place: + +| the cause | the fix | +|---|---| +| a loader misreads its source | fix the loader | +| a published source is wrong, and one better reference covers more than a handful of objects by a stable key | a **supplementary source** (9.4) | +| a published source is wrong for one object, and no rule or reference can produce the value | a **correction** (9.5) | +| a **curated source** (EGC, `bright_stars.csv`, `messier_names.dat`) is wrong or incomplete | edit the file | + +A published source is never edited, even to fix it: it stays as its +publisher released it, so an updated release can be dropped in. + +### 9.3 Matching + +A listing's akas join it to an existing sky object only if both hold: + +1. **The aka parses as a designation of a PiFinder catalog.** Prefixes + are an allowlist (`CATALOG_CODE_ALIASES`, `parse_designation`), so a + double-star discoverer code — `h 301`, `A 58`, `B 53` — never joins + Herschel 400, Abell or Barnard. Matching on a whitespace-stripped + lowercase key is what put ν Aur's names on NGC 4245 (#668). +2. **The match guard passes.** The types are compatible — a star type + (`*`, `D*`, …) joins only a star, and anything else only a non-star — + and the listing lies within 30′ of the sky object, or within the + object's own size if that is larger. + +A rejected match leaves the listing as a sky object of its own — the +right outcome for a double star whose discoverer code once matched a +galaxy. Each rejection is logged, and the import ends with a count. Rejections do not fail the +import. + +**Stars are identified by Hipparcos number.** Each star listing is +cross-matched to `hip_main.dat`: the nearest Hipparcos star within about +3′ whose V is within about 1 mag of the listing's. Listings that match +one HIP number are one sky object, whatever catalogs list them — Str, +SaM, SaR, TLK, RDS and WDS alike — so Mizar is one sky object. A listing +with no match (a SAC double fainter than Hipparcos's completeness, say) +stays its own. A listing whose source position is too wrong to match +(SAC's α Mic, whole degrees off) gets a `hip` correction, which this +stage reads before it cross-matches — the one correction that does not +wait for the final pass (9.5). Source positions are rounded to 0.1ᵐ in RA and 1′ in Dec, +and few sources carry an ID, so position alone or a catalog ID alone +cannot do this. + +A merged star is typed `D*` if SAC's doubles (SaM, less its "rej." +rows), RASC's doubles (RDS), or a WDS pair not flagged optical +(S, U, Y, L) or dubious (X) lists it; otherwise `*`. WDS alone would +make nearly every bright star a double — Vega's WDS companions are +field stars — and Hipparcos's CCDM letter is blank for wide pairs and +for pairs it could not resolve (Sirius, Polaris, Antares), so neither +decides by itself. + +A **not-found** NGC/IC number (Steinicke's status 10, about 300 of +them) keeps its listing, its nominal position and type `?`, and its +description leads with "Not found: nothing at Dreyer's position +(Steinicke)." ahead of Dreyer's text, so nobody hunts an empty field. +Where Corwin identifies one, a correction sets its position and type +and replaces that description. + +For NGC and IC numbers, where the sources disagree about an +identification, Corwin's decides; an identification Corwin leaves open +("?", "??") stays as the sources give it. Corwin is not a vendored +source, so his identifications enter as corrections. + +### 9.4 Supplementary sources + +A supplementary source never adds listings. It is joined on a stable +key and fills or replaces one or more fields by a stated rule: + +| source | joined on | rule | +|---|---|---| +| Hipparcos (`hip_main.dat`) | HIP, from the cross-match | **replace** the position and the primary's V on every matched star listing; a companion's magnitude is untouched | +| IAU WGSN, Catalog of Star Names | HIP | **add** the IAU name as the **preferred name**; older names (Graffias, Tegmen) stay as ordinary names, so text search still finds them | +| Bica et al. 2008 (VizieR J/MNRAS/389/678) | NGC/IC designation | **fill** a missing size on a Magellanic object; never replace one | +| RC3 (VizieR VII/155), extract of the rows whose `BT` is coded "V" | NGC/IC designation | **replace** V with Steinicke's Bmag where that Bmag equals RC3's V-coded `BT` within 0.05 — Steinicke read those as B and subtracted a colour, making 48 galaxies about 1 mag too bright | + +Steinicke's Bmag is never used as a fallback V on its own: for +non-galaxies it is often NGC 2000.0's photographic figure, and for M 43 +it is the exciting star's. An object with no V stays without one unless +a later loader fills it or a correction sets it. + +Fill-only for Bica is deliberate: its cluster-only sizes would otherwise +overwrite the complex sizes PiFinder keeps for objects like NGC 1966. +Where Bica shows that a size is simply wrong (NGC 602's 34′ for 3.2′), +that is a correction citing Bica. + +### 9.5 Corrections + +A **correction** is a cited value that replaces what the sources give, +in `astro_data/corrections/.toml` — one file per published +source it contradicts, so each file doubles as the upstream report. The +format, and every check, is documented in +[`astro_data/corrections/README.md`](../../astro_data/corrections/README.md). + +- **Keyed by designator**, applied to the sky object it resolves to; a + description applies to that one listing. +- **Three operations**: set fields (`obj_type`, `ra`, `dec`, `const`, + `size`, `mag`, `description`); edit names (`names.add`, + `names.remove`, `names.preferred`); correct an identification + (`same_as`, `own_object`, or `hip` for a star listing — applied + before the star cross-match, not in the final pass). +- **Checked.** A field correction states the value it replaces + (`was.`). If the field already holds the corrected value the + correction is redundant; if it holds anything but `was` it is stale. + Either fails the import, as do two corrections reaching one field of + one sky object, a designator that resolves to nothing, and a missing + `why` or `sources`. Name and identification corrections are checked + the same way: removing an absent name, or a `same_as` that is already + true, fails. +- **All or nothing.** Identifications go first, since they change what + designators resolve to; every other correction is then checked before + any is written, and a failure rolls the whole pass back. + +The pass runs last (9.1), so a correction always wins. A rule of thumb +for anyone adding one: if you are about to write more than a handful of +corrections citing the same reference, it is a supplementary source. + +### 9.6 Name order + +The names table holds **common names** and **cross-designations** side +by side, in import order, and nothing marks which is which. The order is +meaningful: the device shows `names[0]` (chart readout, object list) and +the atlas takes the first name that survives its filters. A **preferred +name** is a common name placed first — by WGSN, by the Messier names, or +by a correction. An object without one shows its first cross-designation. + +The name files: + +- `ngc_ic_m/messier/messier_names.dat` is curated, one line per object + (`MName; Name`); its first name is the object's preferred + name. +- `ngc_ic_m/ngc2000/names.dat` is NGC 2000.0's, published. It is read + as published but title-cased on read (the atlas's rule: small words + after the first stay lower case), so "Crab nebula" reads "Crab + Nebula". Its misplaced names are corrected in `ngc2000_names.toml` — + including the half of a fix whose key is another catalog's (the + Horsehead moves from IC 434 to B 33): a correction lives in the file + of the source whose error it fixes. + +The import must therefore keep names in a reproducible order; a `set()` +of strings is ordered by Python's per-process hash seed and is not. + +### 9.7 Committing the database + +The committed `pifinder_objects.db` is about 23 MB compressed, and a +fielded unit's update downloads every version of it that was committed +since its last one. Commit a rebuilt database only in a pull request +that means to change the data, and batch import changes so one rebuild +carries many. Loader code may sit ahead of the committed database in +the meantime — a device never runs the import. The data assertions in +`tests/test_catalog_data.py` read the committed file, so they land +together with the rebuild that makes them true. + +--- + +## 10. Glossary The canonical glossary lives at [`catalog/CONTEXT.md`](./catalog/CONTEXT.md). Use those terms when reading, writing, and discussing code in this area. diff --git a/docs/ax/catalog/CONTEXT.md b/docs/ax/catalog/CONTEXT.md index 4c4ae6be9..613a03f8a 100644 --- a/docs/ax/catalog/CONTEXT.md +++ b/docs/ax/catalog/CONTEXT.md @@ -1,6 +1,6 @@ # Catalog -The Catalog context owns runtime loading, filtering, searching and display of astronomical catalogs (Messier, NGC, IC, WDS, planets, comets…) for the UI and web layers. Catalog state lives in the main UI process; other processes only see catalog content indirectly through the shared `objects.db` and `observations.db`. +The Catalog context owns runtime loading, filtering, searching and display of astronomical catalogs (Messier, NGC, IC, WDS, planets, comets…) for the UI and web layers. Catalog state lives in the main UI process; other processes only see catalog content indirectly through the shared `objects.db` and `observations.db`. It also owns the build-time **catalog import** that produces `objects.db` (see **Import**). > Companion architecture doc: [`../catalog.md`](../catalog.md). @@ -135,6 +135,56 @@ _Avoid_: background loader, async loader. The literal string token pushed onto `ui_queue` once deferred loading completes. The UI watches for it to refresh lists. _Avoid_: load-done event, ready signal. +### Import + +**Catalog import**: +The offline build that turns the **sources** in `astro_data/` into `objects.db`. It runs at development time, not on the device; the device ships the built database. +_Avoid_: catalog load (that's the runtime), DB generation. + +**Source**: +A reference data file under `astro_data/` that a **loader** reads. Most are published (Steinicke, SAC, WDS, Harris…) and vendored as their publisher released them — or as a subset of rows and columns fetched from the publisher by a recorded query (a VizieR extract), so it can be fetched again. A published source can be wrong, and the import never edits one to fix it. The exception is a **curated source**. +_Avoid_: input file, raw data, catalog (a catalog is what the device shows). + +**Curated source**: +A **source** PiFinder compiles itself (EGC, the bright stars list, the Messier names). Its contents are PiFinder's own editorial choices, so it is edited directly — a missing or wrong entry there is fixed in the file, never by a **correction**. +_Avoid_: custom catalog, house list. + +**Loader**: +The import code that reads one **source** and contributes its **catalog listings**, matching each listing to an existing **sky object** or minting a new one. A loader that misreads its source has a bug; one that reads it faithfully does not, even when the source is wrong. +_Avoid_: reader, importer, parser. + +**Supplementary source**: +A **source** read only to fill or replace fields (names, magnitudes, sizes, positions) on sky objects other loaders created, joined on a stable identifier such as HIP. It never adds catalog listings. Use one where a single reference covers more than a handful of objects (IAU WGSN for star names, Bica et al. 2008 for Magellanic sizes). +_Avoid_: enrichment, secondary catalog. + +**Identification**: +Which sky object a **designator** refers to. Sources disagree about identifications (Steinicke and Corwin on IC 2477), and a wrong one either puts a listing on the wrong sky object or splits one thing into two sky objects. A **correction** can fix an identification in either direction: this designator is the same sky object as that one, or it is a sky object of its own. For stars, the Hipparcos (HIP) number is the identity: star listings that cross-match to one HIP number are one sky object, whatever catalogs list them. For NGC and IC numbers, Corwin's identification decides when the sources disagree; where Corwin is undecided ("?", "??"), the sources' identification stands. +_Avoid_: merge, alias, same-as, match (matching is what a **loader** does to arrive at an identification). + +**Not-found object**: +An NGC or IC number with nothing at Dreyer's position (Steinicke's status 10). It stays a sky object — the number still resolves — typed `?` and described as not found, until an identification places it. +_Avoid_: nonexistent object (OpenNGC's `NonEx`; some are later identified), missing object, phantom. + +**Double star** (`D*`): +A star with a companion worth looking for: one an observer's double-star list (SAC, RASC) or a WDS pair that is neither optical nor dubious lists. What the device's "double star" type filter finds. A star that WDS lists only with optical companions (Vega) is a single star (`*`). +_Avoid_: binary (that's physical, and some listed pairs aren't proven binaries), multiple star. + +**Common name**: +A **Name** that is what the sky object is called ("Cigar Galaxy", "Albireo") rather than a designation of it. +_Avoid_: popular name, proper name, alias. + +**Cross-designation**: +A **Name** that is another catalog's designator for the same sky object ("NGC 3034" and "Arp 337" on M 82). The names table holds cross-designations and common names side by side, and nothing marks which is which. +_Avoid_: alias, aka, alternate name. + +**Preferred name**: +The **common name** placed first among a sky object's **Names**, ahead of its cross-designations, so the device and the atlas show it. Only some sky objects have one; for the rest the first name is a cross-designation. Order alone marks it. +_Avoid_: primary name, main name, display name (that's the **catalog display name**). + +**Correction**: +A cited value for one sky object that replaces what the sources give because the sources are wrong. Reserved for what no loader fix or **supplementary source** can produce; the fix of last resort. Keyed by a **designator** and applied to the sky object that designator resolves to — except a description, which belongs to that one **catalog listing**. Two corrections reaching the same field of one sky object through different designators (M 42, NGC 1976) are a conflict, not a precedence question. +_Avoid_: override (it says one value wins, not that the source is wrong), patch, erratum, fix-up. + ### Dynamic catalogs **Dynamic catalog**: @@ -255,3 +305,21 @@ _Avoid_: readonly list, immutable view. > **Dev:** The user loads an observing list with "M 31" and a nameless target at some RA/Dec. What comes out? > > **Domain:** Two **observing list entries**. "M 31" is **catalog-keyed**, so **resolution** finds the existing `CompositeObject` — same `object_id`, same logged state. The RA/Dec one is a **coordinate entry**; it can't resolve, so it's minted as a **coordinate object** with a fresh **virtual ID** under catalog code `OBS`. Don't call either one a "catalog listing" — that's a DB row. + +An import-side dialogue: + +> **Dev:** IC 424 shows as 120′ across. It's 2.5′. I'll fix the Steinicke loader. +> +> **Domain:** Check first whether the **loader** misreads Steinicke. If Steinicke's own file says 120′, the loader is right and the **source** is wrong — so it's a **correction** in `steinicke.toml`, with the value it replaces, why, and a citation. Don't edit Steinicke's file: it's a published source. +> +> **Dev:** And the 49 Magellanic clusters with no size at all? +> +> **Domain:** One reference, Bica 2008, covers all of them by designation. That's a **supplementary source**, not 49 corrections. +> +> **Dev:** NGC 1783 should be a globular. Correction? +> +> **Domain:** No — put it on the EGC list. EGC is a **curated source**: our own list, so we edit it, and an EGC listing makes its sky object a globular. +> +> **Dev:** Last one: M 82 shows "Bode's nebulae". +> +> **Domain:** That's the M 81/82 pair's name, which NGC 2000's names file put on M 82 alone. A correction removes it and makes "Cigar Galaxy" the **preferred name** — the **common name** placed ahead of M 82's **cross-designations**, so the device and the atlas show it. diff --git a/python/PiFinder/catalog_imports/catalog_import_utils.py b/python/PiFinder/catalog_imports/catalog_import_utils.py index 2bd236ac5..51367fd33 100644 --- a/python/PiFinder/catalog_imports/catalog_import_utils.py +++ b/python/PiFinder/catalog_imports/catalog_import_utils.py @@ -181,7 +181,7 @@ def _size_token_to_arcsec(token: str) -> Optional[float]: return None -def parse_arcmin_size(raw: str) -> SizeObject: +def parse_arcmin_size(raw: str, strict: bool = False) -> SizeObject: """Parse a size string into a SizeObject (extents stored in arcseconds). Values default to arcminutes, but a token may override its unit with a @@ -189,7 +189,8 @@ def parse_arcmin_size(raw: str) -> SizeObject: catalogs like EGC ('36"') parse correctly. The axis separators 'x', '×', '/', and '+' are all treated as delimiters, so "32'x6.5'", "0.3/5.8", and "30'+30'" each yield two extents. Tokens whose numeric part can't be - parsed (e.g. "nl", "see") are skipped with a warning. + parsed (e.g. "nl", "see") are skipped with a warning, or raise ValueError + when strict: a correction's size is typed by hand and must parse whole. """ if not raw: return SizeObject([]) @@ -199,6 +200,8 @@ def parse_arcmin_size(raw: str) -> SizeObject: values = [] for token in cleaned.split(): arcsec = _size_token_to_arcsec(token) + if arcsec is None and strict: + raise ValueError(f"non-numeric size token {token!r} in {raw!r}") if arcsec is None: logging.warning("Non-numeric size token %r in %r", token, raw) else: diff --git a/python/PiFinder/catalog_imports/corrections.py b/python/PiFinder/catalog_imports/corrections.py new file mode 100644 index 000000000..faa737538 --- /dev/null +++ b/python/PiFinder/catalog_imports/corrections.py @@ -0,0 +1,948 @@ +""" +Corrections: cited, checked values that replace what a source gives. + +A correction fixes a value that a published source has wrong for one sky +object, where no loader fix or supplementary source can produce the right +one (ADR 0046, docs/ax/catalog.md section 9.5). Corrections live in +astro_data/corrections/.toml, one table per designator; the format +is documented in that directory's README.md. + +load_corrections() reads and validates the files. apply_corrections() is the +import's final pass: it runs after every other stage, so a correction always +has the last word, and it fails the import on any correction that is stale, +redundant, unresolvable or in conflict with another. The one correction it +does not apply is `hip`, which feeds the star cross-match instead: that stage +reads hip_identifications() before it matches. +""" + +import logging +import math +import re +import sqlite3 +import sys +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, cast + +from skyfield.api import load_constellation_names + +import PiFinder.utils as utils +from PiFinder import calc_utils +from PiFinder.composite_object import MagnitudeObject, SizeObject +from PiFinder.db.objects_db import ObjectsDatabase +from PiFinder.obj_types import OBJ_TYPES +from .catalog_import_utils import parse_arcmin_size, parse_designation + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +CORRECTIONS_DIR = Path(utils.astro_data_dir) / "corrections" + +# Fields a correction can set, named like the database columns. A +# description belongs to one catalog listing; the rest to the sky object. +SKY_OBJECT_FIELDS = ("obj_type", "ra", "dec", "const", "size", "mag") +LISTING_FIELDS = ("description",) +FIELDS = SKY_OBJECT_FIELDS + LISTING_FIELDS + +# How close a field must be to count as holding a value, for the `was` +# (stale) and redundancy checks. +MAG_TOLERANCE = 0.05 +POSITION_TOLERANCE_DEG = 1 / 3600 # 1 arcsecond, on RA and Dec separately +SIZE_TOLERANCE_ARCSEC = 1.0 +_FLOAT_SLACK = 1e-9 + +_KEYS = {"why", "sources", "was", "names", "same_as", "own_object", "hip", *FIELDS} +_NAME_KEYS = {"add", "remove", "preferred"} + +# A catalog code, one space, and a number without leading zeros: "NGC 604", +# "Sh2 155". The code must match catalog_objects.catalog_code exactly. +_DESIGNATOR = re.compile(r"([A-Za-z][A-Za-z0-9]*) ([1-9][0-9]*)") + +_CONSTELLATIONS = {abbreviation for abbreviation, _name in load_constellation_names()} + + +class CorrectionError(Exception): + """One or more corrections are invalid, so the import must not finish.""" + + def __init__(self, problems: List[str]): + self.problems = problems + lines = "\n".join(f" {problem}" for problem in problems) + super().__init__(f"{len(problems)} correction problem(s):\n{lines}") + + +@dataclass +class Correction: + """One designator's table from a corrections file, parsed and validated. + + Field values are held in comparable form: RA and Dec in degrees, a size + as its extents in arcseconds, a magnitude as its list of mags. + """ + + source_file: str + designator: str + listing: Tuple[str, int] + why: str + sources: List[str] + fields: Dict[str, Any] = field(default_factory=dict) + was: Dict[str, Any] = field(default_factory=dict) + names_add: List[str] = field(default_factory=list) + names_remove: List[str] = field(default_factory=list) + names_preferred: Optional[str] = None + same_as: Optional[Tuple[str, int]] = None + own_object: bool = False + hip: Optional[int] = None + + @property + def label(self) -> str: + return f"{self.source_file} [{self.designator}]" + + def names_touched(self) -> List[str]: + preferred = [self.names_preferred] if self.names_preferred else [] + return self.names_add + self.names_remove + preferred + + +# ---- parsing values ---- + + +def parse_designator(text: Any) -> Tuple[str, int]: + """'NGC 604' -> ('NGC', 604).""" + match = _DESIGNATOR.fullmatch(text) if isinstance(text, str) else None + if match is None: + raise ValueError( + f"{text!r} is not a designator: write a catalog code, one space and" + ' a number, such as "NGC 604"' + ) + return match.group(1), int(match.group(2)) + + +_SEXAGESIMAL_SEPARATORS = re.compile(r"[\s:hmsd°'′\"″]+") +_UNSIGNED_NUMBER = re.compile(r"[0-9]+(\.[0-9]+)?") + + +def _sexagesimal(text: Any, what: str, example: str) -> Tuple[str, float]: + """Split '-05 23 28' into ('-', 5.391...); 2 or 3 parts, last may be decimal.""" + if not isinstance(text, str): + raise ValueError(f"{what} must be a sexagesimal string, such as {example}") + text = text.strip() + sign = text[0] if text and text[0] in "+-" else "" + parts = [part for part in _SEXAGESIMAL_SEPARATORS.split(text[len(sign) :]) if part] + if not 2 <= len(parts) <= 3 or not all( + _UNSIGNED_NUMBER.fullmatch(part) for part in parts + ): + raise ValueError(f"{what} {text!r} is not sexagesimal, such as {example}") + if any("." in part for part in parts[:-1]): + raise ValueError(f"{what} {text!r}: only its last part may have decimals") + numbers = [float(part) for part in parts] + if any(number >= 60 for number in numbers[1:]): + raise ValueError(f"{what} {text!r}: minutes and seconds must be under 60") + value = sum(number / 60**i for i, number in enumerate(numbers)) + return sign, value + + +def parse_ra(text: Any) -> float: + """Sexagesimal hours ('05 35 17.3', '05:35:17.3', '5h35m17.3s') to degrees.""" + sign, hours = _sexagesimal(text, "ra", '"05 35 17.3"') + if sign or hours >= 24: + raise ValueError(f"ra {text!r} must be unsigned hours under 24") + return hours * 15 + + +def parse_dec(text: Any) -> float: + """Sexagesimal degrees ('-05 23 28', '+41°16′09″') to degrees.""" + sign, degrees = _sexagesimal(text, "dec", '"-05 23 28"') + if degrees > 90: + raise ValueError(f"dec {text!r} is beyond the pole") + return -degrees if sign == "-" else degrees + + +def _is_number(value: Any) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + ) + + +def _parse_obj_type(value: Any) -> str: + if value not in OBJ_TYPES: + raise ValueError( + f"obj_type {value!r} is not a type code; use one of " + ", ".join(OBJ_TYPES) + ) + return value + + +def _parse_const(value: Any) -> str: + if value not in _CONSTELLATIONS: + raise ValueError(f"const {value!r} is not an IAU constellation abbreviation") + return value + + +def _parse_size(value: Any) -> List[float]: + if not isinstance(value, str): + raise ValueError(f'size must be a string, such as "2.5\'", not {value!r}') + extents = cast(List[float], parse_arcmin_size(value, strict=True).extents) + if not extents: + raise ValueError(f"size {value!r} holds no size") + return extents + + +def _parse_mag(value: Any) -> List[float]: + values = value if isinstance(value, list) else [value] + if not values or not all(_is_number(mag) for mag in values): + raise ValueError( + f"mag must be a number, or a list of numbers for a variable's range," + f" not {value!r}" + ) + return [float(mag) for mag in values] + + +def _parse_description(value: Any) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError("description must be a non-empty string") + return value.strip() + + +_FIELD_PARSERS: Dict[str, Callable[[Any], Any]] = { + "obj_type": _parse_obj_type, + "ra": parse_ra, + "dec": parse_dec, + "const": _parse_const, + "size": _parse_size, + "mag": _parse_mag, + "description": _parse_description, +} + +# Fields that can be empty in the database, and so can be `was = ""`. +_EMPTY: Dict[str, Callable[[], Any]] = { + "size": list, + "mag": list, + "description": str, +} + + +def _parse_was(field_name: str, value: Any) -> Any: + if value == "" and field_name in _EMPTY: + return _EMPTY[field_name]() + return _FIELD_PARSERS[field_name](value) + + +# ---- comparing and showing values ---- + + +def _same_number(a: Any, b: Any, tolerance: float) -> bool: + if _is_number(a) and _is_number(b): + return abs(a - b) <= tolerance + _FLOAT_SLACK + return a == b + + +def _same_list(a: List[Any], b: List[Any], tolerance: float) -> bool: + return len(a) == len(b) and all(_same_number(x, y, tolerance) for x, y in zip(a, b)) + + +def same_value(field_name: str, a: Any, b: Any) -> bool: + """Whether two values of a field agree, within that field's tolerance.""" + if field_name in ("ra", "dec"): + if not (_is_number(a) and _is_number(b)): + return a == b + difference = abs(a - b) + if field_name == "ra": + difference = min(difference, 360 - difference) + return difference <= POSITION_TOLERANCE_DEG + _FLOAT_SLACK + if field_name == "size": + return _same_list(a, b, SIZE_TOLERANCE_ARCSEC) + if field_name == "mag": + return _same_list(a, b, MAG_TOLERANCE) + return a == b + + +def _format_ra(degrees: float) -> str: + hundredths = round(degrees / 15 * 360000) + hours, rest = divmod(hundredths, 360000) + minutes, rest = divmod(rest, 6000) + return f"{hours:02d} {minutes:02d} {rest / 100:05.2f}" + + +def _format_dec(degrees: float) -> str: + sign = "-" if degrees < 0 else "+" + tenths = round(abs(degrees) * 36000) + whole, rest = divmod(tenths, 36000) + minutes, rest = divmod(rest, 600) + return f"{sign}{whole:02d} {minutes:02d} {rest / 10:04.1f}" + + +def format_value(field_name: str, value: Any) -> str: + """A value as it would be written in a corrections file.""" + if value in ("", []): + return '""' + if field_name == "ra" and _is_number(value): + return f'"{_format_ra(value)}"' + if field_name == "dec" and _is_number(value): + return f'"{_format_dec(value)}"' + if field_name == "size": + return repr(SizeObject(value).to_display_string()) + if field_name == "mag": + return str(value[0]) if len(value) == 1 else str(value) + return repr(value) + + +# ---- loading ---- + + +def load_corrections(directory: Path = CORRECTIONS_DIR) -> List[Correction]: + """Read and validate every corrections file, in file-name order. + + Raises CorrectionError listing every problem found in any file. + """ + corrections: List[Correction] = [] + problems: List[str] = [] + for path in sorted(Path(directory).glob("*.toml")): + try: + with open(path, "rb") as file: + document = tomllib.load(file) + except tomllib.TOMLDecodeError as error: + problems.append(f"{path.name}: not valid TOML: {error}") + continue + for designator, table in document.items(): + try: + corrections.append(_parse_correction(path.name, designator, table)) + except CorrectionError as error: + problems.extend(error.problems) + if problems: + raise CorrectionError(problems) + logging.info("Loaded %d corrections from %s", len(corrections), directory) + return corrections + + +def _string_list(value: Any) -> Optional[List[str]]: + """The stripped strings of a non-empty list of non-empty strings, or None.""" + if not isinstance(value, list) or not value: + return None + if not all(isinstance(item, str) and item.strip() for item in value): + return None + return [item.strip() for item in value] + + +def _parse_correction(source_file: str, designator: str, table: Any) -> Correction: + label = f"{source_file} [{designator}]" + problems: List[str] = [] + + def problem(message: str) -> None: + problems.append(f"{label}: {message}") + + if not isinstance(table, dict): + raise CorrectionError( + [f"{label}: must be a table, with the designator quoted in its header"] + ) + try: + listing = parse_designator(designator) + except ValueError as error: + problem(str(error)) + listing = ("", 0) + + for key in sorted(set(table) - _KEYS): + problem(f"unknown key {key!r}") + + why = table.get("why") + if not isinstance(why, str) or not why.strip(): + problem("needs `why`: what is wrong, and what the sources say") + why = "" + sources = _string_list(table.get("sources")) + if sources is None: + problem("needs `sources`: a list of citations (URLs where they exist)") + sources = [] + + correction = Correction(source_file, designator, listing, why.strip(), sources) + + for field_name in FIELDS: + if field_name in table: + try: + correction.fields[field_name] = _FIELD_PARSERS[field_name]( + table[field_name] + ) + except ValueError as error: + problem(str(error)) + + was = table.get("was", {}) + if not isinstance(was, dict): + problem('`was` must be a table of fields, such as was.size = "120\'"') + was = {} + for field_name in was: + if field_name not in FIELDS: + problem(f"was.{field_name} is not a field") + elif field_name not in table: + problem(f"was.{field_name} without a {field_name} to correct") + else: + try: + correction.was[field_name] = _parse_was(field_name, was[field_name]) + except ValueError as error: + problem(f"was.{field_name}: {error}") + for field_name, value in correction.fields.items(): + if field_name not in was: + problem( + f"{field_name} needs was.{field_name}, the value it replaces" + ' ("" if the field is empty)' + ) + elif field_name in correction.was and same_value( + field_name, value, correction.was[field_name] + ): + problem(f"{field_name} equals was.{field_name}, so it corrects nothing") + + _parse_names(table.get("names"), correction, problem) + _parse_identification(table, correction, problem) + + if not ( + correction.fields + or correction.names_touched() + or correction.same_as + or correction.own_object + or correction.hip is not None + or problems + ): + problem("corrects nothing") + if problems: + raise CorrectionError(problems) + return correction + + +def _parse_names( + names: Any, correction: Correction, problem: Callable[[str], None] +) -> None: + if names is None: + return + if not isinstance(names, dict): + problem("`names` must be a table: names.add, names.remove, names.preferred") + return + for key in sorted(set(names) - _NAME_KEYS): + problem(f"unknown key names.{key}") + for key in ("add", "remove"): + if key in names: + values = _string_list(names[key]) + if values is None: + problem(f"names.{key} must be a non-empty list of names") + elif len(set(values)) != len(values): + problem(f"names.{key} lists a name twice") + else: + setattr(correction, f"names_{key}", values) + if "preferred" in names: + preferred = names["preferred"] + if not isinstance(preferred, str) or not preferred.strip(): + problem("names.preferred must be a name") + else: + correction.names_preferred = preferred.strip() + overlap = set(correction.names_add) & set(correction.names_remove) + if overlap: + problem(f"names.add and names.remove both list {sorted(overlap)}") + preferred = correction.names_preferred + if preferred in correction.names_add: + problem(f"names.preferred adds {preferred!r} already; drop it from names.add") + if preferred in correction.names_remove: + problem(f"names.preferred {preferred!r} is also in names.remove") + + +def _parse_identification( + table: Dict[str, Any], correction: Correction, problem: Callable[[str], None] +) -> None: + if "same_as" in table: + try: + correction.same_as = parse_designator(table["same_as"]) + except ValueError as error: + problem(f"same_as: {error}") + else: + if correction.same_as == correction.listing: + problem("same_as names the designator itself") + if "own_object" in table: + if table["own_object"] is not True: + problem("own_object can only be true") + else: + correction.own_object = True + if correction.same_as and correction.own_object: + problem("same_as and own_object contradict each other") + if "hip" in table: + hip = table["hip"] + if isinstance(hip, bool) or not isinstance(hip, int) or hip <= 0: + problem(f"hip must be a Hipparcos number, not {hip!r}") + else: + correction.hip = hip + + +def hip_identifications(corrections: Iterable[Correction]) -> Dict[str, int]: + """Map each designator with a `hip` correction to its Hipparcos number. + + For the star-identity stage, which uses these in place of position when it + cross-matches star listings to Hipparcos. The final pass leaves `hip` + alone, since by then the cross-match has already run. + """ + identifications: Dict[str, int] = {} + labels: Dict[str, str] = {} + problems = [] + for correction in corrections: + if correction.hip is None: + continue + if correction.designator in identifications: + problems.append( + f"{correction.label}: conflicts with {labels[correction.designator]}:" + " both set hip" + ) + continue + identifications[correction.designator] = correction.hip + labels[correction.designator] = correction.label + if problems: + raise CorrectionError(problems) + return identifications + + +# ---- the final pass ---- + + +def apply_corrections(db: ObjectsDatabase, corrections: List[Correction]) -> None: + """The import's final pass: apply every correction, or none. + + Identifications go first, splits (own_object) before joins (same_as), + since they change what designators resolve to. Then every field and name + correction is checked against the database before any is written. Any + problem rolls the pass back and raises CorrectionError. + """ + conn = db.conn + try: + _apply(conn.cursor(), corrections) + except BaseException: + conn.rollback() + raise + conn.commit() + + +def _apply(cursor: sqlite3.Cursor, corrections: List[Correction]) -> None: + moved = _apply_identifications(cursor, corrections) + planned = _plan(cursor, corrections) + for correction, listing_id, object_id in planned: + _write(cursor, correction, listing_id, object_id) + _recompute_constellations(cursor, planned) + _check_nothing_left_behind(cursor, moved) + logging.info( + "Applied %d corrections (hip identifications are left to the star" + " cross-match: %d)", + len(corrections), + sum(correction.hip is not None for correction in corrections), + ) + + +def _designator(listing: Tuple[str, int]) -> str: + return f"{listing[0]} {listing[1]}" + + +def resolves_to(name: str, listing: Tuple[str, int]) -> bool: + """Whether a loader's lookup would resolve this name to this listing. + + The question ObjectFinder.get_object_id asks: the lowercased name is the + listing's exact key ("NGC70"), or parse_designation reads it as the + listing ("Cr 42" is Col 42, "NGC 0070" is NGC 70). "H 24" and "M 2-9" + resolve to nothing, since neither prefix names a catalog. + """ + code, sequence = listing + return ( + name.lower() == f"{code.lower()}{sequence}" + or parse_designation(name) == listing + ) + + +def _is_own_name(name: str, correction: Correction) -> bool: + """Whether a name is the correction's designator, as its listing's own. + + Every loader writes a listing's own name as " ", which the + lookup can't resolve for codes that are no designation prefix (B, H, WDS), + so that form counts as well as any form the lookup resolves. + """ + return name == correction.designator or resolves_to(name, correction.listing) + + +def _resolve( + cursor: sqlite3.Cursor, listing: Tuple[str, int] +) -> Tuple[Optional[sqlite3.Row], str]: + """The catalog_objects row a designator names, or None and why not.""" + rows = cursor.execute( + "SELECT id, object_id FROM catalog_objects" + " WHERE catalog_code = ? AND sequence = ?", + listing, + ).fetchall() + if len(rows) == 1: + return rows[0], "" + if rows: + return None, f"{_designator(listing)} is listed {len(rows)} times" + known = cursor.execute( + "SELECT 1 FROM catalogs WHERE catalog_code = ?", (listing[0],) + ).fetchone() + if known is None: + return None, f"no catalog has the code {listing[0]!r}" + return None, f"{_designator(listing)} resolves to nothing" + + +def _names(cursor: sqlite3.Cursor, object_id: int) -> List[sqlite3.Row]: + return cursor.execute( + "SELECT id, common_name, origin FROM names WHERE object_id = ? ORDER BY id", + (object_id,), + ).fetchall() + + +def _listings(cursor: sqlite3.Cursor, object_id: int) -> List[sqlite3.Row]: + return cursor.execute( + "SELECT id, catalog_code, sequence FROM catalog_objects" + " WHERE object_id = ? ORDER BY id", + (object_id,), + ).fetchall() + + +def _move_names( + cursor: sqlite3.Cursor, rows: Iterable[sqlite3.Row], object_id: int +) -> None: + """Move name rows to the end of another sky object's names. + + Re-inserted rather than re-pointed, so a moved name can't land ahead of + the names already there; a name the sky object already has is dropped. + """ + present = {row["common_name"] for row in _names(cursor, object_id)} + for row in rows: + cursor.execute("DELETE FROM names WHERE id = ?", (row["id"],)) + if row["common_name"] not in present: + cursor.execute( + "INSERT INTO names (object_id, common_name, origin) VALUES (?, ?, ?)", + (object_id, row["common_name"], row["origin"]), + ) + present.add(row["common_name"]) + + +def _move_listing( + cursor: sqlite3.Cursor, + correction: Correction, + listing_id: int, + from_object: int, + to_object: int, +) -> None: + """Move one catalog listing, and its own names that are its designator.""" + cursor.execute( + "UPDATE catalog_objects SET object_id = ? WHERE id = ?", (to_object, listing_id) + ) + own_names = [ + row + for row in _names(cursor, from_object) + if row["origin"] == correction.listing[0] + and _is_own_name(row["common_name"], correction) + ] + _move_names(cursor, own_names, to_object) + if not any( + _is_own_name(row["common_name"], correction) + for row in _names(cursor, to_object) + ): + cursor.execute( + "INSERT INTO names (object_id, common_name, origin) VALUES (?, ?, ?)", + (to_object, correction.designator, correction.listing[0]), + ) + + +def _apply_identifications( + cursor: sqlite3.Cursor, corrections: List[Correction] +) -> List[Tuple[Correction, int]]: + """Apply own_object and same_as; return each with the sky object it left.""" + identifications = [c for c in corrections if c.own_object or c.same_as] + problems: List[str] = [] + claimed: Dict[Tuple[str, int], Correction] = {} + for correction in identifications: + other = claimed.setdefault(correction.listing, correction) + if other is not correction: + problems.append( + f"{correction.label}: conflicts with {other.label}: both correct" + f" the identification of {correction.designator}" + ) + joining = {c.listing for c in identifications if c.same_as} + for correction in identifications: + if correction.same_as in joining: + problems.append( + f"{correction.label}: same_as {_designator(correction.same_as)}," + " which is itself corrected by a same_as; point" + f" {correction.designator} at the sky object it ends up on" + ) + if problems: + raise CorrectionError(problems) + + moved: List[Tuple[Correction, int]] = [] + splits_first = sorted(identifications, key=lambda c: not c.own_object) + for correction in splits_first: + listing, why_not = _resolve(cursor, correction.listing) + if listing is None: + problems.append(f"{correction.label}: {why_not}") + continue + source = listing["object_id"] + siblings = len(_listings(cursor, source)) - 1 + if correction.own_object: + if not siblings: + problems.append( + f"{correction.label}: own_object, but {correction.designator}" + " is already a sky object of its own" + ) + continue + cursor.execute( + "INSERT INTO objects" + " (obj_type, ra, dec, const, size, mag, surface_brightness)" + " SELECT obj_type, ra, dec, const, size, mag, surface_brightness" + " FROM objects WHERE id = ?", + (source,), + ) + new_object = cursor.lastrowid + assert new_object is not None + _move_listing(cursor, correction, listing["id"], source, new_object) + else: + assert correction.same_as is not None + target_designator = _designator(correction.same_as) + target, why_not = _resolve(cursor, correction.same_as) + if target is None: + problems.append(f"{correction.label}: same_as: {why_not}") + continue + if target["object_id"] == source: + problems.append( + f"{correction.label}: same_as {target_designator}, but it is" + " already the same sky object" + ) + continue + if siblings: + _move_listing( + cursor, correction, listing["id"], source, target["object_id"] + ) + else: + # The whole sky object is this listing: merge it in. + cursor.execute( + "UPDATE catalog_objects SET object_id = ? WHERE id = ?", + (target["object_id"], listing["id"]), + ) + _move_names(cursor, _names(cursor, source), target["object_id"]) + cursor.execute( + "DELETE FROM object_images WHERE object_id = ?", (source,) + ) + cursor.execute("DELETE FROM objects WHERE id = ?", (source,)) + moved.append((correction, source)) + if problems: + raise CorrectionError(problems) + return moved + + +def _plan( + cursor: sqlite3.Cursor, corrections: List[Correction] +) -> List[Tuple[Correction, int, int]]: + """Resolve every correction and check it, before anything is written. + + Returns (correction, listing id, sky object id) for each. + """ + problems: List[str] = [] + planned: List[Tuple[Correction, int, int]] = [] + claims: Dict[Tuple[Any, ...], Correction] = {} + conflicts: Dict[Tuple[str, str], List[str]] = defaultdict(list) + for correction in corrections: + listing, why_not = _resolve(cursor, correction.listing) + if listing is None: + problems.append(f"{correction.label}: {why_not}") + continue + listing_id, object_id = listing["id"], listing["object_id"] + planned.append((correction, listing_id, object_id)) + + claimed: List[Tuple[Tuple[Any, ...], str]] = [] + for field_name in correction.fields: + if field_name in SKY_OBJECT_FIELDS: + claimed.append(((object_id, field_name), field_name)) + else: + claimed.append((("listing", listing_id, field_name), field_name)) + if correction.names_preferred: + claimed.append(((object_id, "names.preferred"), "names.preferred")) + for name in correction.names_touched(): + claimed.append(((object_id, "name", name), f"the name {name!r}")) + if correction.hip is not None: + claimed.append((("listing", listing_id, "hip"), "hip")) + for key, what in claimed: + other = claims.setdefault(key, correction) + if other is not correction: + conflicts[(correction.label, other.label)].append(what) + for (label, other_label), whats in conflicts.items(): + problems.append( + f"{label}: conflicts with {other_label}: both correct " + + ", ".join(dict.fromkeys(whats)) + + " of the same sky object" + ) + for correction, listing_id, object_id in planned: + problems.extend(_check(cursor, correction, listing_id, object_id)) + if problems: + raise CorrectionError(problems) + return planned + + +def _current_values( + cursor: sqlite3.Cursor, listing_id: int, object_id: int +) -> Dict[str, Any]: + sky_object = cursor.execute( + "SELECT * FROM objects WHERE id = ?", (object_id,) + ).fetchone() + listing = cursor.execute( + "SELECT description FROM catalog_objects WHERE id = ?", (listing_id,) + ).fetchone() + return { + "obj_type": sky_object["obj_type"] or "", + "ra": sky_object["ra"], + "dec": sky_object["dec"], + "const": sky_object["const"] or "", + "size": SizeObject.from_json(sky_object["size"]).extents, + "mag": MagnitudeObject.from_json(sky_object["mag"]).mags, + "description": (listing["description"] or "").strip(), + } + + +def _check( + cursor: sqlite3.Cursor, correction: Correction, listing_id: int, object_id: int +) -> List[str]: + """The stale and redundant checks, for fields and names alike.""" + problems = [] + label = correction.label + current = _current_values(cursor, listing_id, object_id) + for field_name, value in correction.fields.items(): + holds = current[field_name] + if same_value(field_name, holds, value): + problems.append( + f"{label}: {field_name} is already {format_value(field_name, value)};" + " the correction is redundant, so delete it" + ) + elif not same_value(field_name, holds, correction.was[field_name]): + problems.append( + f"{label}: {field_name} is stale: was.{field_name} says" + f" {format_value(field_name, correction.was[field_name])}, but the" + f" database holds {format_value(field_name, holds)}; check the" + " correction against the source" + ) + names = [row["common_name"] for row in _names(cursor, object_id)] + for name in correction.names_remove: + if name not in names: + problems.append( + f"{label}: names.remove {name!r}, which the sky object doesn't have" + ) + for name in correction.names_add: + if name in names: + problems.append(f"{label}: names.add {name!r}, which it already has") + if names and correction.names_preferred == names[0]: + problems.append( + f"{label}: names.preferred {names[0]!r} is already the first name" + ) + return problems + + +def _write( + cursor: sqlite3.Cursor, correction: Correction, listing_id: int, object_id: int +) -> None: + for field_name, value in correction.fields.items(): + if field_name == "description": + cursor.execute( + "UPDATE catalog_objects SET description = ? WHERE id = ?", + (value, listing_id), + ) + continue + if field_name == "size": + # The extents are corrected; the position angle stays as it was. + stored = cursor.execute( + "SELECT size FROM objects WHERE id = ?", (object_id,) + ).fetchone()["size"] + position_angle = SizeObject.from_json(stored).position_angle + value = SizeObject(value, position_angle=position_angle).to_json() + elif field_name == "mag": + value = MagnitudeObject(list(value)).to_json() + cursor.execute( + f"UPDATE objects SET {field_name} = ? WHERE id = ?", (value, object_id) + ) + + code = correction.listing[0] + for name in correction.names_remove: + cursor.execute( + "DELETE FROM names WHERE object_id = ? AND common_name = ?", + (object_id, name), + ) + for name in correction.names_add: + cursor.execute( + "INSERT INTO names (object_id, common_name, origin) VALUES (?, ?, ?)", + (object_id, name, code), + ) + preferred = correction.names_preferred + if preferred: + # Names are ordered by row id, so putting one first means re-inserting + # them all, the preferred name's rows ahead of the rest. + rows = [ + (row["common_name"], row["origin"]) for row in _names(cursor, object_id) + ] + first = [row for row in rows if row[0] == preferred] or [(preferred, code)] + rest = [row for row in rows if row[0] != preferred] + cursor.execute("DELETE FROM names WHERE object_id = ?", (object_id,)) + cursor.executemany( + "INSERT INTO names (object_id, common_name, origin) VALUES (?, ?, ?)", + [(object_id, name, origin) for name, origin in first + rest], + ) + + +def _recompute_constellations( + cursor: sqlite3.Cursor, planned: List[Tuple[Correction, int, int]] +) -> None: + """A moved sky object's constellation follows its position, as at insert, + unless a correction sets const itself (a boundary case like NGC 1404).""" + moved: Dict[int, str] = {} + const_set: Set[int] = set() + for correction, _listing_id, object_id in planned: + if "ra" in correction.fields or "dec" in correction.fields: + moved.setdefault(object_id, correction.designator) + if "const" in correction.fields: + const_set.add(object_id) + for object_id, designator in moved.items(): + if object_id in const_set: + continue + row = cursor.execute( + "SELECT ra, dec, const FROM objects WHERE id = ?", (object_id,) + ).fetchone() + const = calc_utils.sf_utils.radec_to_constellation(row["ra"], row["dec"]) + logging.info( + "Constellation of %s follows its corrected position: %s -> %s", + designator, + row["const"], + const, + ) + cursor.execute("UPDATE objects SET const = ? WHERE id = ?", (const, object_id)) + + +def _check_nothing_left_behind( + cursor: sqlite3.Cursor, moved: List[Tuple[Correction, int]] +) -> None: + """Fail a split or join whose old sky object still answers to its designator. + + Another catalog's name that the loaders' lookup resolves to the moved + designator means that catalog's listing was matched through the wrong + identification, and it stays behind on the old sky object, or that its + record asserts the cross-identification the correction denies (Steinicke's + NGC records carry the "M nn" names). Runs after names corrections, so a + stray name can be removed by one. + """ + problems = [] + for correction, source in moved: + stray = [ + row + for row in _names(cursor, source) + if resolves_to(row["common_name"], correction.listing) + ] + if not stray: + continue + operation = "own_object" if correction.own_object else "same_as" + left = ", ".join( + _designator((row["catalog_code"], row["sequence"])) + for row in _listings(cursor, source) + ) + found = ", ".join( + f"{row['common_name']!r} (from {row['origin']})" for row in stray + ) + problems.append( + f"{correction.label}: {operation} leaves {left} still named {found};" + " if that catalog's listing joined through the wrong identification," + " correct it too, and if its record only names the designator," + " remove the name with names.remove" + ) + if problems: + raise CorrectionError(problems) diff --git a/python/PiFinder/catalog_imports/main.py b/python/PiFinder/catalog_imports/main.py index 7fcaa5bae..f8530e475 100644 --- a/python/PiFinder/catalog_imports/main.py +++ b/python/PiFinder/catalog_imports/main.py @@ -10,6 +10,7 @@ from PiFinder import timez from .catalog_import_utils import print_database, resolve_object_images +from .corrections import apply_corrections, load_corrections from .database import init_shared_database # Loader registry - import functions dynamically to reduce coupling @@ -81,6 +82,10 @@ def main(): logging.info("Starting") + # Read the corrections up front, so a malformed file fails the import + # before the loaders run rather than after. + corrections = load_corrections() + # Initialize shared databases logging.info("Creating DB") objects_db, _ = init_shared_database() @@ -119,6 +124,11 @@ def main(): logging.error(f"Failed to run {function_name} from {module_name}: {e}") raise + # The final pass: corrections have the last word over every stage above, + # so any stage added later goes before this (ADR 0046). + logging.info("Applying corrections...") + apply_corrections(objects_db, corrections) + # Populate the images table logging.info("Resolving object images...") resolve_object_images() diff --git a/python/requirements_dev.txt b/python/requirements_dev.txt index 7e864ae70..4ed41993a 100644 --- a/python/requirements_dev.txt +++ b/python/requirements_dev.txt @@ -18,6 +18,9 @@ types-aiofiles types-tqdm==4.65.0 pandas-stubs types-waitress +# The catalog import reads astro_data/corrections/*.toml; tomllib is stdlib from +# 3.11. The device never runs the import, so this is a dev dependency only. +tomli==2.4.1; python_version < "3.11" # Test-only: drift guard for the .pifinder JSON Schema (test_pifinder_schema.py) jsonschema==4.23.0 # Pin to avoid pyobjc 12.0 which has macOS 15 build issues diff --git a/python/tests/test_catalog_corrections.py b/python/tests/test_catalog_corrections.py new file mode 100644 index 000000000..2af662b1b --- /dev/null +++ b/python/tests/test_catalog_corrections.py @@ -0,0 +1,854 @@ +"""Tests for the catalog import's corrections: loading, checks, and the final pass. + +Everything runs against a small objects DB built here, never the committed +one. See ADR 0046 and astro_data/corrections/README.md for the format. +""" + +import json +import textwrap + +import pytest + +from PiFinder.catalog_imports.corrections import ( + CorrectionError, + apply_corrections, + hip_identifications, + load_corrections, + parse_dec, + parse_designator, + parse_ra, + resolves_to, +) +from PiFinder.db.objects_db import ObjectsDatabase + +pytestmark = pytest.mark.unit + +CITED = 'why = "the source is wrong"\nsources = ["https://example.org/ref"]\n' + + +def _size(*arcsec, pa=0.0): + return json.dumps({"e": list(arcsec), "p": pa}) + + +def _mag(*mags): + return json.dumps({"mags": list(mags), "filter_mag": mags[0] if mags else 99}) + + +@pytest.fixture +def db(tmp_path): + """Sky objects, each with its listings and names (in insertion order). + + As in the real import, an "M nn" name comes with Steinicke's NGC record, + so its origin is NGC, not M. + """ + db = ObjectsDatabase(tmp_path / "objects.db") + db.create_tables() + for code in ("M", "NGC", "IC", "C", "SaM", "Col", "H", "B"): + db.insert_catalog(code, 10000, code) + objects = [ + # M 42 / NGC 1976 at 05 35 17.1, -05 23 25.0 + ( + ("Nb", 83.82125, -5.390277777777778, "Ori", _size(5400, 3600), _mag(4.0)), + [("M", 42, "Orion Nebula"), ("NGC", 1976, "Theta Orionis neb")], + [("NGC 1976", "NGC"), ("Orion Nebula", "NGC"), ("M 42", "NGC")], + ), + ( + ("Nb", 83.405, -0.413, "Ori", _size(7200), _mag()), + [("IC", 424, "F, eL")], + [("IC 424", "IC")], + ), + ( + ("Gx", 148.975, 69.683, "UMa", _size(672, 258, pa=65.0), _mag(8.4)), + [("NGC", 3034, "vB, vL"), ("M", 82, "Cigar")], + [ + ("NGC 3034", "NGC"), + ("Bode's nebulae", "NGC"), + ("M82", "NGC"), + ("M 82", "NGC"), + ], + ), + ( + ("?", 120.0, -40.0, "Pup", _size(), _mag()), + [("IC", 2477, "Not found")], + [("IC 2477", "IC")], + ), + ( + ("Gx", 120.1, -40.1, "Pup", _size(60), _mag(13.0)), + [("IC", 2480, "pF, S")], + [("IC 2480", "IC")], + ), + ( + ("Gx", 345.514, 27.053, "Peg", _size(90, 30, pa=26.0), _mag(13.5)), + [("NGC", 7466, "eF, eS"), ("IC", 5281, "= NGC 7466?")], + [("NGC 7466", "NGC"), ("IC 5281", "IC")], + ), + ( + ("D*", 315.32, -33.78, "Mic", _size(), _mag(4.9)), + [("SaM", 1297, "alpha Mic")], + [("SaM 1297", "SaM")], + ), + ] + for values, listings, names in objects: + object_id = db.insert_object(*values) + for code, sequence, description in listings: + db.insert_catalog_object(object_id, code, sequence, description) + for name, origin in names: + db.insert_name(object_id, name, origin) + yield db + db.conn.close() + + +@pytest.fixture +def corrections_dir(tmp_path): + path = tmp_path / "corrections" + path.mkdir() + return path + + +def write(directory, text, name="steinicke.toml"): + (directory / name).write_text(textwrap.dedent(text)) + + +def load(directory, text, name="steinicke.toml"): + write(directory, text, name) + return load_corrections(directory) + + +def problems_of(excinfo): + return "\n".join(excinfo.value.problems) + + +def sky_object(db, code, sequence): + return db.conn.execute( + "SELECT o.* FROM objects o JOIN catalog_objects co ON co.object_id = o.id" + " WHERE co.catalog_code = ? AND co.sequence = ?", + (code, sequence), + ).fetchone() + + +def object_id(db, code, sequence): + return sky_object(db, code, sequence)["id"] + + +def names(db, code, sequence): + return [ + row["common_name"] + for row in db.conn.execute( + "SELECT common_name FROM names WHERE object_id = ? ORDER BY id", + (object_id(db, code, sequence),), + ) + ] + + +def description(db, code, sequence): + return db.conn.execute( + "SELECT description FROM catalog_objects" + " WHERE catalog_code = ? AND sequence = ?", + (code, sequence), + ).fetchone()["description"] + + +def snapshot(db): + return [ + [tuple(row) for row in db.conn.execute(f"SELECT * FROM {table} ORDER BY id")] + for table in ("objects", "catalog_objects", "names") + ] + + +def apply(db, directory, text, name="steinicke.toml"): + apply_corrections(db, load(directory, text, name)) + + +def apply_fails(db, directory, text, name="steinicke.toml"): + corrections = load(directory, text, name) + before = snapshot(db) + with pytest.raises(CorrectionError) as excinfo: + apply_corrections(db, corrections) + assert snapshot(db) == before, "a failed pass must change nothing" + return problems_of(excinfo) + + +# ---- values ---- + + +@pytest.mark.parametrize( + "text", ["05 35 17.3", "05:35:17.3", "5h35m17.3s", "05 35.28833333"] +) +def test_parse_ra_reads_sexagesimal_hours(text): + assert parse_ra(text) == pytest.approx(83.8220833, abs=1e-6) + + +@pytest.mark.parametrize( + "text, degrees", + [ + ("-05 23 28", -5.3911111), + ("+41°16′09″", 41.2691667), + ("41 16 09", 41.2691667), + ("-00 30 00", -0.5), + ("-00:30", -0.5), + ], +) +def test_parse_dec_reads_sexagesimal_degrees_and_keeps_the_sign(text, degrees): + assert parse_dec(text) == pytest.approx(degrees, abs=1e-6) + + +@pytest.mark.parametrize( + "text", [83.82, "83.82", "24 00 00", "-05 35 17", "05 61 00", "05.5 30", "5 hrs"] +) +def test_parse_ra_rejects(text): + with pytest.raises(ValueError): + parse_ra(text) + + +@pytest.mark.parametrize("text", [-5.39, "91 00 00", "-05 23 28 1", "-05 23 60"]) +def test_parse_dec_rejects(text): + with pytest.raises(ValueError): + parse_dec(text) + + +def test_parse_designator(): + assert parse_designator("NGC 604") == ("NGC", 604) + assert parse_designator("Sh2 155") == ("Sh2", 155) + for bad in ("NGC604", "NGC 0604", "NGC 604", "NGC 604A", "M 0", 604): + with pytest.raises(ValueError): + parse_designator(bad) + + +# ---- loading ---- + + +def test_load_parses_values_into_reference_units(corrections_dir): + (correction,) = load( + corrections_dir, + """ + ["IC 424"] + why = "Steinicke's 120' is the whole complex" + sources = ["https://example.org/a", "https://example.org/b"] + size = "2.5'" + was.size = "120'" + mag = [4.86, 7.32] + was.mag = "" + dec = "-00 24 47" + was.dec = "-00 24 50" + """, + ) + assert correction.designator == "IC 424" + assert correction.listing == ("IC", 424) + assert correction.label == "steinicke.toml [IC 424]" + assert correction.sources == ["https://example.org/a", "https://example.org/b"] + assert correction.fields["size"] == pytest.approx([150.0]) + assert correction.was["size"] == pytest.approx([7200.0]) + assert correction.fields["mag"] == [4.86, 7.32] + assert correction.was["mag"] == [] + assert correction.fields["dec"] == pytest.approx(-(24 / 60 + 47 / 3600)) + + +@pytest.mark.parametrize( + "body, expected", + [ + ('sources = ["https://x"]\nmag = 1\nwas.mag = 2', "needs `why`"), + ('why = "x"\nmag = 1\nwas.mag = 2', "needs `sources`"), + ('why = "x"\nsources = []\nmag = 1\nwas.mag = 2', "needs `sources`"), + (CITED + 'size = "2.5\'"', "size needs was.size"), + (CITED + 'mag = 1\nwas.mag = 2\nwas.size = "1\'"', "was.size without a size"), + (CITED + "mag = 1\nwas.mag = 2\nwas.magnitude = 2", "was.magnitude is not"), + (CITED + "magnitude = 5", "unknown key 'magnitude'"), + (CITED + 'obj_type = "Galaxy"\nwas.obj_type = "Nb"', "not a type code"), + (CITED + 'obj_type = "Gx"\nwas.obj_type = ""', "was.obj_type"), + (CITED + 'const = "Orion"\nwas.const = "Ori"', "not an IAU constellation"), + (CITED + 'size = "2.5\' x see"\nwas.size = ""', "non-numeric size token"), + (CITED + 'size = 2.5\nwas.size = ""', "size must be a string"), + (CITED + 'mag = "11.4"\nwas.mag = ""', "mag must be a number"), + (CITED + 'mag = true\nwas.mag = ""', "mag must be a number"), + (CITED + 'ra = 83.4\nwas.ra = "05 33 37"', "sexagesimal string"), + (CITED + 'size = "2.5\'"\nwas.size = "150\\""', "corrects nothing"), + (CITED + "mag = 4.0\nwas.mag = 4.04", "corrects nothing"), + (CITED + 'same_as = "IC 2480"\nown_object = true', "contradict"), + (CITED + "own_object = false", "can only be true"), + (CITED + 'same_as = "IC 424"', "the designator itself"), + (CITED + 'same_as = "IC424"', "same_as: 'IC424' is not a designator"), + (CITED + "names.add = []", "names.add must be a non-empty list"), + (CITED + 'names.add = ["A", "A"]', "lists a name twice"), + (CITED + 'names.rename = ["A"]', "unknown key names.rename"), + (CITED + 'names.add = ["A"]\nnames.remove = ["A"]', "both list ['A']"), + (CITED + 'names.add = ["A"]\nnames.preferred = "A"', "drop it from names"), + (CITED + 'names.remove = ["A"]\nnames.preferred = "A"', "also in names.r"), + (CITED + "hip = 0", "Hipparcos number"), + (CITED + 'hip = "102989"', "Hipparcos number"), + (CITED + "hip = true", "Hipparcos number"), + (CITED, "corrects nothing"), + ], +) +def test_load_rejects(corrections_dir, body, expected): + with pytest.raises(CorrectionError) as excinfo: + load(corrections_dir, '["IC 424"]\n' + body) + assert expected in problems_of(excinfo) + assert "steinicke.toml [IC 424]" in problems_of(excinfo) + + +def test_load_rejects_a_header_that_is_not_a_designator(corrections_dir): + with pytest.raises(CorrectionError) as excinfo: + load(corrections_dir, '["IC424"]\n' + CITED + "hip = 5") + assert "'IC424' is not a designator" in problems_of(excinfo) + + +def test_load_rejects_keys_outside_any_table(corrections_dir): + with pytest.raises(CorrectionError) as excinfo: + load(corrections_dir, CITED) + assert "must be a table" in problems_of(excinfo) + + +def test_load_rejects_invalid_toml(corrections_dir): + with pytest.raises(CorrectionError) as excinfo: + load(corrections_dir, "[IC 424]\n" + CITED) + assert "steinicke.toml: not valid TOML" in problems_of(excinfo) + + +def test_load_reports_every_problem_in_every_file(corrections_dir): + write(corrections_dir, '["IC 424"]\nmag = 1\nwas.mag = 2\n', "a.toml") + write(corrections_dir, '["M 42"]\n' + CITED + "magnitude = 1\n", "b.toml") + with pytest.raises(CorrectionError) as excinfo: + load_corrections(corrections_dir) + text = problems_of(excinfo) + assert "a.toml [IC 424]: needs `why`" in text + assert "a.toml [IC 424]: needs `sources`" in text + assert "b.toml [M 42]: unknown key 'magnitude'" in text + + +def test_load_reads_files_in_name_order_and_ignores_other_files(corrections_dir): + write(corrections_dir, '["M 42"]\n' + CITED + "hip = 2\n", "b.toml") + write(corrections_dir, '["IC 424"]\n' + CITED + "hip = 1\n", "a.toml") + (corrections_dir / "README.md").write_text("# not a correction") + corrections = load_corrections(corrections_dir) + assert [c.designator for c in corrections] == ["IC 424", "M 42"] + + +def test_load_with_no_corrections(tmp_path, corrections_dir): + assert load_corrections(corrections_dir) == [] + assert load_corrections(tmp_path / "missing") == [] + + +def test_shipped_corrections_load(): + load_corrections() + + +# ---- hip ---- + + +def test_hip_is_exposed_for_the_star_cross_match_and_not_applied(db, corrections_dir): + before = snapshot(db) + corrections = load( + corrections_dir, + '["SaM 1297"]\n' + CITED + "hip = 102989\n", + "sac_multistars.toml", + ) + assert hip_identifications(corrections) == {"SaM 1297": 102989} + apply_corrections(db, corrections) + assert snapshot(db) == before + + +def test_hip_twice_for_one_designator_fails(db, corrections_dir): + write(corrections_dir, '["SaM 1297"]\n' + CITED + "hip = 1\n", "a.toml") + write(corrections_dir, '["SaM 1297"]\n' + CITED + "hip = 2\n", "b.toml") + corrections = load_corrections(corrections_dir) + with pytest.raises(CorrectionError) as excinfo: + hip_identifications(corrections) + assert "b.toml [SaM 1297]: conflicts with a.toml [SaM 1297]" in problems_of(excinfo) + with pytest.raises(CorrectionError): + apply_corrections(db, corrections) + + +def test_hip_designator_must_resolve(db, corrections_dir): + problems = apply_fails(db, corrections_dir, '["SaM 9"]\n' + CITED + "hip = 5\n") + assert "SaM 9 resolves to nothing" in problems + + +# ---- setting fields ---- + + +def test_size_correction_keeps_the_position_angle(db, corrections_dir): + apply( + db, + corrections_dir, + '["M 82"]\n' + CITED + "size = \"12'x5'\"\nwas.size = \"11.2'x4.3'\"\n", + ) + size = json.loads(sky_object(db, "M", 82)["size"]) + assert size["e"] == pytest.approx([720.0, 300.0]) + assert size["p"] == 65.0 + + +def test_mag_takes_a_number_or_a_range(db, corrections_dir): + apply( + db, + corrections_dir, + '["IC 424"]\n' + CITED + 'mag = 11.4\nwas.mag = ""\n' + '["IC 2480"]\n' + CITED + "mag = [4.86, 7.32]\nwas.mag = 13.0\n", + ) + assert json.loads(sky_object(db, "IC", 424)["mag"]) == { + "mags": [11.4], + "filter_mag": 11.4, + } + mag = json.loads(sky_object(db, "IC", 2480)["mag"]) + assert mag["mags"] == [4.86, 7.32] + assert mag["filter_mag"] == pytest.approx(6.09) + + +def test_a_field_correction_reaches_the_sky_object_through_any_designator( + db, corrections_dir +): + apply( + db, + corrections_dir, + '["NGC 1976"]\n' + CITED + 'obj_type = "C+N"\nwas.obj_type = "Nb"\n', + ) + assert sky_object(db, "M", 42)["obj_type"] == "C+N" + + +def test_a_description_belongs_to_one_listing(db, corrections_dir): + apply( + db, + corrections_dir, + '["NGC 1976"]\n' + + CITED + + 'description = """\nThe Orion Nebula.\n"""\n' + + 'was.description = "Theta Orionis neb"\n', + ) + assert description(db, "NGC", 1976) == "The Orion Nebula." + assert description(db, "M", 42) == "Orion Nebula" + + +def test_was_description_can_be_empty(db, corrections_dir): + db.conn.execute("UPDATE catalog_objects SET description = '' WHERE sequence = 424") + db.conn.commit() + apply( + db, + corrections_dir, + '["IC 424"]\n' + CITED + 'description = "Faint."\nwas.description = ""\n', + ) + assert description(db, "IC", 424) == "Faint." + + +def test_a_position_correction_moves_the_constellation(db, corrections_dir, caplog): + caplog.set_level("INFO") + apply( + db, + corrections_dir, + '["IC 424"]\n' + + CITED + + 'ra = "06 31 55"\nwas.ra = "05 33 37.2"\n' + + 'dec = "+04 56 34"\nwas.dec = "-00 24 46.8"\n', + ) + row = sky_object(db, "IC", 424) + assert row["ra"] == pytest.approx(97.979167, abs=1e-6) + assert row["dec"] == pytest.approx(4.942778, abs=1e-6) + assert row["const"] == "Mon" + assert "Constellation of IC 424 follows its corrected position: Ori -> Mon" in ( + caplog.text + ) + + +def test_a_const_correction_wins_over_the_position(db, corrections_dir): + apply( + db, + corrections_dir, + '["IC 424"]\n' + + CITED + + 'ra = "06 31 55"\nwas.ra = "05 33 37.2"\n' + + 'const = "Eri"\nwas.const = "Ori"\n', + ) + assert sky_object(db, "IC", 424)["const"] == "Eri" + + +def test_redundant_correction_fails(db, corrections_dir): + problems = apply_fails( + db, corrections_dir, '["M 42"]\n' + CITED + "mag = 4.0\nwas.mag = 5.0\n" + ) + assert "steinicke.toml [M 42]: mag is already 4.0" in problems + assert "redundant" in problems + + +def test_stale_correction_fails_and_says_what_the_database_holds(db, corrections_dir): + problems = apply_fails( + db, + corrections_dir, + '["IC 424"]\n' + CITED + 'size = "2.5\'"\nwas.size = "60\'"\n', + ) + assert "size is stale: was.size says '1°'" in problems + assert "the database holds '2°'" in problems + + +@pytest.mark.parametrize( + "field, was, value, matches", + [ + ("mag", "4.04", "9.0", True), + ("mag", "4.06", "9.0", False), + ("ra", '"05 35 17.15"', '"05 35 20"', True), # 0.75 arcsec of RA + ("ra", '"05 35 17.2"', '"05 35 20"', False), # 1.5 arcsec + ("dec", '"-05 23 25.9"', '"-05 23 40"', True), + ("dec", '"-05 23 26.1"', '"-05 23 40"', False), + ("size", "\"90'x60'\"", "\"80'x60'\"", True), + ("size", '"1.5°x1°"', "\"80'x60'\"", True), + ("size", "\"90'x61'\"", "\"80'x60'\"", False), + ("size", '"90\'"', "\"80'x60'\"", False), + ], +) +def test_was_tolerances(db, corrections_dir, field, was, value, matches): + text = f'["M 42"]\n{CITED}{field} = {value}\nwas.{field} = {was}\n' + if matches: + apply(db, corrections_dir, text) + else: + assert f"{field} is stale" in apply_fails(db, corrections_dir, text) + + +def test_redundancy_is_within_tolerance_too(db, corrections_dir): + problems = apply_fails( + db, corrections_dir, '["M 42"]\n' + CITED + "mag = 4.03\nwas.mag = 6\n" + ) + assert "mag is already 4.03" in problems + + +@pytest.mark.parametrize( + "designator, expected", + [("NGC 9999", "NGC 9999 resolves to nothing"), ("Xyz 1", "no catalog has")], +) +def test_a_designator_that_resolves_to_nothing_fails( + db, corrections_dir, designator, expected +): + problems = apply_fails( + db, corrections_dir, f'["{designator}"]\n{CITED}mag = 1\nwas.mag = 2\n' + ) + assert expected in problems + + +def test_two_corrections_reaching_one_field_of_a_sky_object_fail(db, corrections_dir): + write(corrections_dir, '["M 42"]\n' + CITED + "mag = 4.1\nwas.mag = 4.0\n") + problems = apply_fails( + db, + corrections_dir, + '["NGC 1976"]\n' + CITED + "mag = 4.2\nwas.mag = 4.0\n", + "other.toml", + ) + assert ( + "steinicke.toml [M 42]: conflicts with other.toml [NGC 1976]: both" + " correct mag" in problems + ) + + +def test_two_corrections_on_different_fields_of_a_sky_object_are_fine( + db, corrections_dir +): + write(corrections_dir, '["M 42"]\n' + CITED + "mag = 4.1\nwas.mag = 4.0\n") + apply( + db, + corrections_dir, + '["NGC 1976"]\n' + CITED + "size = \"85'x60'\"\nwas.size = \"90'x60'\"\n", + "other.toml", + ) + row = sky_object(db, "M", 42) + assert json.loads(row["mag"])["mags"] == [4.1] + assert json.loads(row["size"])["e"] == pytest.approx([5100.0, 3600.0]) + + +# ---- names ---- + + +def test_names_remove_and_preferred(db, corrections_dir): + apply( + db, + corrections_dir, + '["M 82"]\n' + + CITED + + 'names.remove = ["Bode\'s nebulae"]\n' + + 'names.preferred = "Cigar Galaxy"\n', + "ngc2000_names.toml", + ) + assert names(db, "M", 82) == ["Cigar Galaxy", "NGC 3034", "M82", "M 82"] + + +def test_names_preferred_moves_an_existing_name_first(db, corrections_dir): + apply(db, corrections_dir, '["M 42"]\n' + CITED + 'names.preferred = "M 42"\n') + assert names(db, "M", 42) == ["M 42", "NGC 1976", "Orion Nebula"] + + +def test_names_add_appends(db, corrections_dir): + apply(db, corrections_dir, '["M 42"]\n' + CITED + 'names.add = ["Great Nebula"]\n') + assert names(db, "NGC", 1976)[-1] == "Great Nebula" + + +def test_names_remove_takes_every_copy(db, corrections_dir): + db.insert_name(object_id(db, "M", 82), "M82", "Arp") + apply(db, corrections_dir, '["NGC 3034"]\n' + CITED + 'names.remove = ["M82"]\n') + assert "M82" not in names(db, "M", 82) + + +@pytest.mark.parametrize( + "operation, expected", + [ + ('names.remove = ["Crab Nebula"]', "which the sky object doesn't have"), + ('names.add = ["Orion Nebula"]', "which it already has"), + ('names.preferred = "NGC 1976"', "is already the first name"), + ], +) +def test_name_checks(db, corrections_dir, operation, expected): + problems = apply_fails(db, corrections_dir, f'["M 42"]\n{CITED}{operation}\n') + assert expected in problems + + +def test_two_corrections_touching_one_name_fail(db, corrections_dir): + write(corrections_dir, '["M 42"]\n' + CITED + 'names.add = ["Great Nebula"]\n') + problems = apply_fails( + db, + corrections_dir, + '["NGC 1976"]\n' + CITED + 'names.preferred = "Great Nebula"\n', + "other.toml", + ) + assert "both correct names.preferred" not in problems + assert "both correct the name 'Great Nebula'" in problems + + +# ---- identifications ---- + + +def test_same_as_merges_a_lone_listing_into_the_other_sky_object(db, corrections_dir): + orphan = object_id(db, "IC", 2477) + apply(db, corrections_dir, '["IC 2477"]\n' + CITED + 'same_as = "IC 2480"\n') + assert object_id(db, "IC", 2477) == object_id(db, "IC", 2480) + assert names(db, "IC", 2480) == ["IC 2480", "IC 2477"] + assert description(db, "IC", 2477) == "Not found" + assert ( + db.conn.execute("SELECT 1 FROM objects WHERE id = ?", (orphan,)).fetchone() + is None + ) + + +def test_same_as_moves_one_listing_off_a_shared_sky_object(db, corrections_dir): + apply(db, corrections_dir, '["IC 5281"]\n' + CITED + 'same_as = "IC 2480"\n') + assert object_id(db, "IC", 5281) == object_id(db, "IC", 2480) + assert names(db, "IC", 5281) == ["IC 2480", "IC 5281"] + assert names(db, "NGC", 7466) == ["NGC 7466"] + + +def test_a_field_correction_follows_its_listing_after_same_as(db, corrections_dir): + apply( + db, + corrections_dir, + '["IC 2477"]\n' + + CITED + + 'same_as = "IC 2480"\n' + + 'description = "= IC 2480 (Corwin)"\nwas.description = "Not found"\n', + ) + assert description(db, "IC", 2477) == "= IC 2480 (Corwin)" + assert description(db, "IC", 2480) == "pF, S" + + +def test_same_as_that_is_already_true_fails(db, corrections_dir): + problems = apply_fails( + db, corrections_dir, '["NGC 1976"]\n' + CITED + 'same_as = "M 42"\n' + ) + assert "already the same sky object" in problems + + +def test_same_as_an_unresolved_target_fails(db, corrections_dir): + problems = apply_fails( + db, corrections_dir, '["IC 2477"]\n' + CITED + 'same_as = "IC 9999"\n' + ) + assert "same_as: IC 9999 resolves to nothing" in problems + + +def test_same_as_chains_fail(db, corrections_dir): + problems = apply_fails( + db, + corrections_dir, + '["IC 2477"]\n' + + CITED + + 'same_as = "IC 424"\n' + + '["IC 424"]\n' + + CITED + + 'same_as = "IC 2480"\n', + ) + assert "[IC 2477]: same_as IC 424, which is itself corrected" in problems + + +def test_two_identifications_of_one_listing_conflict(db, corrections_dir): + write(corrections_dir, '["IC 5281"]\n' + CITED + "own_object = true\n") + problems = apply_fails( + db, + corrections_dir, + '["IC 5281"]\n' + CITED + 'same_as = "IC 2480"\n', + "other.toml", + ) + assert "both correct the identification of IC 5281" in problems + + +def test_own_object_splits_a_listing_into_a_sky_object_of_its_own(db, corrections_dir): + apply( + db, + corrections_dir, + '["IC 5281"]\n' + + CITED + + "own_object = true\n" + + 'obj_type = "D*"\nwas.obj_type = "Gx"\n' + + "mag = 12.1\nwas.mag = 13.5\n", + ) + assert object_id(db, "IC", 5281) != object_id(db, "NGC", 7466) + split = sky_object(db, "IC", 5281) + assert split["obj_type"] == "D*" + assert json.loads(split["mag"])["mags"] == [12.1] + assert split["ra"] == pytest.approx(345.514) # copied, since not corrected + assert names(db, "IC", 5281) == ["IC 5281"] + assert names(db, "NGC", 7466) == ["NGC 7466"] + assert sky_object(db, "NGC", 7466)["obj_type"] == "Gx" + + +def test_same_as_a_split_designator_joins_its_new_sky_object(db, corrections_dir): + apply( + db, + corrections_dir, + '["IC 2477"]\n' + + CITED + + 'same_as = "IC 5281"\n' + + '["IC 5281"]\n' + + CITED + + "own_object = true\n", + ) + assert object_id(db, "IC", 2477) == object_id(db, "IC", 5281) + assert object_id(db, "IC", 5281) != object_id(db, "NGC", 7466) + + +def test_own_object_that_is_already_true_fails(db, corrections_dir): + problems = apply_fails( + db, corrections_dir, '["IC 424"]\n' + CITED + "own_object = true\n" + ) + assert "already a sky object of its own" in problems + + +def test_a_split_that_leaves_a_listing_behind_fails(db, corrections_dir): + # A Caldwell listing that joined through "IC5281" stays on NGC 7466. + ngc_7466 = object_id(db, "NGC", 7466) + db.insert_catalog_object(ngc_7466, "C", 99, "") + db.insert_name(ngc_7466, "C 99", "C") + db.insert_name(ngc_7466, "IC5281", "C") + problems = apply_fails( + db, corrections_dir, '["IC 5281"]\n' + CITED + "own_object = true\n" + ) + assert ( + "[IC 5281]: own_object leaves NGC 7466, C 99 still named 'IC5281' (from C)" + in problems + ) + + +def test_a_stray_name_left_behind_can_be_removed_by_a_correction(db, corrections_dir): + db.insert_name(object_id(db, "NGC", 7466), "IC 5281", "NGC") + apply( + db, + corrections_dir, + '["IC 5281"]\n' + + CITED + + "own_object = true\n" + + '["NGC 7466"]\n' + + CITED + + 'names.remove = ["IC 5281"]\n', + ) + assert names(db, "NGC", 7466) == ["NGC 7466"] + assert names(db, "IC", 5281) == ["IC 5281"] + + +@pytest.mark.parametrize("identification", ["own_object = true", 'same_as = "IC 2480"']) +def test_moving_an_m_listing_needs_the_ngc_records_m_name_removed( + db, corrections_dir, identification +): + # Steinicke's NGC record carries "M 42", so moving the M listing alone + # leaves NGC 1976 asserting a cross-identification the correction denies. + move = f'["M 42"]\n{CITED}{identification}\n' + problems = apply_fails(db, corrections_dir, move) + assert "leaves NGC 1976 still named 'M 42' (from NGC)" in problems + + apply( + db, + corrections_dir, + move + '["NGC 1976"]\n' + CITED + 'names.remove = ["M 42"]\n', + ) + assert object_id(db, "M", 42) != object_id(db, "NGC", 1976) + assert names(db, "NGC", 1976) == ["NGC 1976", "Orion Nebula"] + assert names(db, "M", 42)[-1] == "M 42" + + +@pytest.mark.parametrize( + "name, listing, resolves", + [ + ("Cr 42", ("Col", 42), True), + ("Collinder 42", ("Col", 42), True), + ("NGC 0070", ("NGC", 70), True), + ("ngc 70", ("NGC", 70), True), + ("NGC70", ("NGC", 70), True), # the lookup's exact key + ("H24", ("H", 24), True), # likewise + ("H 24", ("H", 24), False), # a bare "h" names no catalog + ("M 2-9", ("M", 29), False), # Minkowski 2-9, not Messier 29 + ("M 2-9", ("M", 9), False), + ("NGC 7466", ("NGC", 70), False), + ], +) +def test_resolves_to_asks_the_loaders_lookup_question(name, listing, resolves): + assert resolves_to(name, listing) is resolves + + +def _share_with_ngc_7466(db, code, sequence, own_name, stray): + """Put a listing, its own name and another catalog's name on NGC 7466.""" + ngc_7466 = object_id(db, "NGC", 7466) + db.insert_catalog_object(ngc_7466, code, sequence, "") + db.insert_name(ngc_7466, own_name, code) + if stray: + db.insert_name(ngc_7466, stray, "SaM") + + +@pytest.mark.parametrize( + "code, sequence, stray, left_behind", + [ + ("Col", 42, "Cr 42", True), + ("NGC", 70, "NGC 0070", True), + ("H", 24, "H 24", False), + ("M", 29, "M 2-9", False), + ], +) +def test_a_name_left_behind_counts_if_the_lookup_resolves_it( + db, corrections_dir, code, sequence, stray, left_behind +): + _share_with_ngc_7466(db, code, sequence, f"{code} {sequence}", stray) + split = f'["{code} {sequence}"]\n{CITED}own_object = true\n' + if left_behind: + problems = apply_fails(db, corrections_dir, split) + assert f"still named {stray!r} (from SaM)" in problems + else: + apply(db, corrections_dir, split) + assert object_id(db, code, sequence) != object_id(db, "NGC", 7466) + assert names(db, "NGC", 7466) == ["NGC 7466", "IC 5281", stray] + + +@pytest.mark.parametrize( + "code, sequence, own_name", + [ + ("B", 33, "B 33"), # the lookup can't resolve a bare "B" + ("Col", 42, "Collinder 42"), # the lookup resolves the alias + ], +) +def test_a_split_takes_the_listings_own_names( + db, corrections_dir, code, sequence, own_name +): + _share_with_ngc_7466(db, code, sequence, own_name, None) + apply(db, corrections_dir, f'["{code} {sequence}"]\n{CITED}own_object = true\n') + assert names(db, code, sequence) == [own_name] + assert names(db, "NGC", 7466) == ["NGC 7466", "IC 5281"] + + +def test_a_failed_check_rolls_back_the_identifications_too(db, corrections_dir): + apply_fails( + db, + corrections_dir, + '["IC 2477"]\n' + + CITED + + 'same_as = "IC 2480"\n' + + '["M 42"]\n' + + CITED + + "mag = 4.1\nwas.mag = 9.0\n", + ) + assert object_id(db, "IC", 2477) != object_id(db, "IC", 2480)