From 36c7f492926e3c236dab8bc0e8ecb0d964ae0216 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 22 Oct 2026 20:05:52 -0700 Subject: [PATCH] fix(catalogs): join akas by allowlisted designation, and four import fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracted from mrosseel's #379 (squash commit 954a83c46 on nixos) with his agreement. The Perek-Kohoutek catalog stays in #379. The changed lines are identical to his, so they drop out when #379 rebases. - ObjectFinder fell back to ui_utils.normalize(), which strips spaces and hyphens. That turned a double star's Herschel number ("H 24", "h 289") into a Herschel 400 key, so SAC multiple-star and red-star listings joined H400 galaxies and clusters (#668). It also turned a compound designation ("M 2-9") into another catalog's sequence (M 29). parse_designation() now splits an aka into a catalog code and a sequence, and accepts only the prefixes in CATALOG_CODE_ALIASES. - That allowlist maps "Cr" to Col, so M 45's "Cr 42" aka now joins the Collinder 42 sky object, where the import used to build a second Pleiades. add_missing_messier_objects() sets the curated Pleiades position, magnitude and size on that shared object, because insert() leaves a joined sky object unchanged. - load_sharpless computed j_dec_deg but passed the B1950 declination, so all 313 Sharpless objects sat 0.26 to 0.37 degrees off in declination. - abell.tsv: the ngc6742 alias belongs on row 50, not row 51 (#668). Row 47 had lost its declination sign (-00 13 51 per SIMBAD). - load_taas200 and load_wds kept names in a set(), whose order follows Python's per-process hash seed. They keep the source order now, so two imports give the same names and the same TAAS links. Ta2 52 joins NGC 2237, which TAAS lists before NGC 2244. parse_designation also reads long-form and zero-padded designations ("Collinder 463", "NGC 0497"). So 67 Lyngå listings now join their Collinder objects, and 33 Arp listings their NGC or IC galaxies, where each used to be a duplicate sky object. Arp 127, 276 and 282 move from their IC galaxy to the NGC galaxy the source lists first. This is code only: the committed database is unchanged. The rebuilt database lands later, in one rebuild with the other catalog-audit fixes. Co-authored-by: mrosseel Co-Authored-By: Claude Opus 5.5 --- astro_data/abell.tsv | 6 +- .../catalog_imports/catalog_import_utils.py | 88 ++++++++++++++++++- .../catalog_imports/post_processing.py | 9 ++ .../catalog_imports/specialized_loaders.py | 10 ++- python/PiFinder/catalog_imports/wds_loader.py | 5 +- python/tests/test_designation_parsing.py | 86 ++++++++++++++++++ 6 files changed, 192 insertions(+), 12 deletions(-) create mode 100644 python/tests/test_designation_parsing.py diff --git a/astro_data/abell.tsv b/astro_data/abell.tsv index 80f4cb0ab..317aa892f 100644 --- a/astro_data/abell.tsv +++ b/astro_data/abell.tsv @@ -41,11 +41,11 @@ catalog_identifier name long_name ra dec magnitude major_axis Type Const 44 Abell 44 277.5470833 -16.75666667 15.4 0.45 PN Sgr 45 Abell 45 277.5670833 -11.61555556 12.8 2.4 PN Sct 46 Abell 46 277.8275 26.93638889 14.3 0.55 PN Lyr -47 Abell 47 278.8441667 0.2305555556 19.5 0.15 PN Ser +47 Abell 47 278.8441667 -0.2305555556 19.5 0.15 PN Ser 48 Abell 48 280.6954167 -3.221388889 17.2 0.35 PN Aql 49 Abell 49 283.3691667 -6.476388889 16.1 0.3 PN Sct -50 Abell 50 284.8329167 48.46472222 13.4 0.3 PN Dra -51 Abell 51 ngc6742 285.2558333 -18.20388889 15 0.55 PN Sgr +50 Abell 50 ngc6742 284.8329167 48.46472222 13.4 0.3 PN Dra +51 Abell 51 285.2558333 -18.20388889 15 0.55 PN Sgr 52 Abell 52 286.135 17.95222222 16.5 0.3 PN Aql 53 Abell 53 286.6920833 6.397222222 15.5 0.25 PN Aql 54 Abell 54 287.165 22.98305556 17.1 0.45 PN Vul diff --git a/python/PiFinder/catalog_imports/catalog_import_utils.py b/python/PiFinder/catalog_imports/catalog_import_utils.py index e828e37b2..2bd236ac5 100644 --- a/python/PiFinder/catalog_imports/catalog_import_utils.py +++ b/python/PiFinder/catalog_imports/catalog_import_utils.py @@ -6,12 +6,11 @@ import logging import re -from typing import Dict, Optional +from typing import Dict, Optional, Tuple from dataclasses import dataclass, field from tqdm import tqdm from PiFinder.composite_object import MagnitudeObject, SizeObject -from PiFinder.ui.ui_utils import normalize from PiFinder import calc_utils from PiFinder.db.objects_db import ObjectsDatabase from PiFinder.db.observations_db import ObservationsDatabase @@ -150,7 +149,10 @@ def get_object_id(self, object_name: str): logging.debug(f"Looking up object id for {object_name}") result = self.mappings.get(object_name.lower()) if not result: - result = self.mappings.get(normalize(object_name)) + parsed = parse_designation(object_name) + if parsed is not None: + catalog_code, sequence = parsed + result = self.mappings.get(f"{catalog_code.lower()}{sequence}") if result: logging.debug(f"Found object id {result} for {object_name}") else: @@ -230,6 +232,86 @@ def trim_string(s): return " ".join(s.split()) +# Designation prefixes that name a PiFinder catalog, keyed on the lowercase, +# whitespace-free prefix as source catalogs write it. A prefix must be listed +# here for an alias to link one object to another; every other designation is +# kept as a plain name. The same catalog is spelled several ways across +# sources, and SIMBAD writes Abell planetaries as "PN A66 nn". +# +# Deliberately absent: +# - A bare "h". Herschel 400 uses catalog code "H", but "H 1-1" and similar +# names are Haro planetary nebulae, and a bare "H 12" can also be Hubble. +# - A bare "a" or "b". Double-star sources use them for the discoverers +# Aitken and van den Bos ("A 58", "B 53"). A loader whose source means +# Abell or Barnard passes them as extra_aliases to parse_designation. +# - "wds" and "pk". Their designations end in digits that are not a sequence +# ("WDS 00001+7508", "PK 120+09.1"). +CATALOG_CODE_ALIASES: Dict[str, str] = { + "abell": "Abl", + "abl": "Abl", + "pna66": "Abl", + "arp": "Arp", + "barnard": "B", + "c": "C", + "caldwell": "C", + "col": "Col", + "collinder": "Col", + "cr": "Col", + "egc": "EGC", + "har": "Har", + "herschel": "H", + "ic": "IC", + "lyn": "Lyn", + "m": "M", + "messier": "M", + "ngc": "NGC", + "rds": "RDS", + "saa": "SaA", + "sam": "SaM", + "sar": "SaR", + "sh2": "Sh2", + "sharpless": "Sh2", + "str": "Str", + "ta2": "Ta2", + "tlk": "TLK", +} + +# A designation reads "": the sequence is the trailing run of +# digits, the prefix is everything before it. +_DESIGNATION_RE = re.compile(r"(.*?)\s*(\d+)") + + +def parse_designation( + raw: str, extra_aliases: Optional[Dict[str, str]] = None +) -> Optional[Tuple[str, int]]: + """Split a source designation into a PiFinder catalog code and sequence. + + "NGC 7008" -> ("NGC", 7008), "Sh 2-176" -> ("Sh2", 176), + "PN A66 80" -> ("Abl", 80). + + Returns None when the designation names no PiFinder catalog. That covers + the planetary-nebula families whose numeric part is compound: "M 1-92" + (Minkowski), "H 3-29" (Haro), "K 2-1" (Kohoutek), "NGC 650-1" (the two + halves of M76). Those leave a prefix of "m1", "h3", "k2" or "ngc650", + none of which is a catalog, so they are rejected instead of being read as + Messier 92, Herschel 29, "K 21" or NGC 6501. + + ``extra_aliases`` adds prefixes for one source, for example {"a": "Abl"} + for a source that writes Abell planetaries as "A 50". + """ + match = _DESIGNATION_RE.fullmatch(trim_string(raw).lower()) + if match is None: + return None + prefix, sequence = match.groups() + key = prefix.rstrip("-.").replace(" ", "") + catalog_code = CATALOG_CODE_ALIASES.get(key) + if catalog_code is None and extra_aliases: + catalog_code = extra_aliases.get(key) + if catalog_code is None: + return None + return catalog_code, int(sequence) + + def delete_catalog_from_database(catalog_code: str): """Delete a catalog and all its related records from the database""" if objects_db is None: diff --git a/python/PiFinder/catalog_imports/post_processing.py b/python/PiFinder/catalog_imports/post_processing.py index d411547ce..651369216 100644 --- a/python/PiFinder/catalog_imports/post_processing.py +++ b/python/PiFinder/catalog_imports/post_processing.py @@ -148,6 +148,15 @@ def add_missing_messier_objects(): aka_names=m45_aka_names, ) m45.insert() + # "Cr 42" links M45 to the Collinder 42 object, which insert() does not + # change. Set the Pleiades values above on that shared object. + objects_db.update_object_by_id( + m45.object_id, + ra=m45.ra, + dec=m45.dec, + mag=m45.mag.to_json(), + size=m45.size.to_json(), + ) # M24 - Sagittarius Star Cloud (no NGC equivalent, it's a dense star field) # RA: 18h 18m 24s = 274.6°, Dec: -18° 24′ 00″ = -18.4° diff --git a/python/PiFinder/catalog_imports/specialized_loaders.py b/python/PiFinder/catalog_imports/specialized_loaders.py index e8d68aee1..19ca1782e 100644 --- a/python/PiFinder/catalog_imports/specialized_loaders.py +++ b/python/PiFinder/catalog_imports/specialized_loaders.py @@ -302,8 +302,10 @@ def load_taas200(): extra_desc = "\n" + "; ".join(extra) desc += extra_desc - duplicate_names = set(other_catalog) - duplicate_names.add(other_names) + # Keep the source order: the first name that resolves decides + # which sky object this entry joins, so the import stays + # reproducible. + duplicate_names = list(dict.fromkeys([*other_catalog, other_names])) new_object = NewCatalogObject( object_type=obj_type, catalog_code=catalog, @@ -313,7 +315,7 @@ def load_taas200(): mag=mag, size=size, description=desc, - aka_names=list(duplicate_names), + aka_names=duplicate_names, ) objects_to_insert.append(new_object) @@ -578,7 +580,7 @@ def load_sharpless(): catalog_code=catalog, sequence=record["Sh2"], ra=j_ra_deg, - dec=dec_deg, + dec=j_dec_deg, size=SizeObject.from_arcmin(float(record["Diam"])), mag=MagnitudeObject([]), description=desc, diff --git a/python/PiFinder/catalog_imports/wds_loader.py b/python/PiFinder/catalog_imports/wds_loader.py index 395f35ec0..30a541d5a 100644 --- a/python/PiFinder/catalog_imports/wds_loader.py +++ b/python/PiFinder/catalog_imports/wds_loader.py @@ -175,7 +175,8 @@ def parse_coordinates_arcsec(coord): return ra_to_deg(ra_h, ra_m, ra_s), dec_to_deg(dec_deg, dec_m, dec_s) def handle_multiples(key, values) -> dict: - discoverers = set() + # Ordered, so the names keep the source order in every import. + discoverers: dict = {} components = {} result = {} descriptions = [] @@ -208,7 +209,7 @@ def handle_multiples(key, values) -> dict: result["mag"] = MagnitudeObject([mag1, mag2]) sizemax = float(np.max([value["Sep_First"], value["Sep_Last"]])) result["size"] = SizeObject.from_arcsec(round(sizemax, 1)) - discoverers.add(value["Discoverer_Number"]) + discoverers[value["Discoverer_Number"]] = None notes = value["Notes"].strip() notes_str = "" if len(notes) == 0 else f" Notes: {notes}" components = value["Components"].strip() diff --git a/python/tests/test_designation_parsing.py b/python/tests/test_designation_parsing.py new file mode 100644 index 000000000..9be1646be --- /dev/null +++ b/python/tests/test_designation_parsing.py @@ -0,0 +1,86 @@ +"""Designation parsing for catalog imports. + +A designation with a compound numeric part ("M 1-92", a Minkowski planetary +nebula) must not collapse into a different catalog entry (Messier 92). The +Perek-Kohoutek name column has 188 Minkowski names, so a parser that drops +the hyphen links 145 of them to the wrong object. +""" + +import pytest + +from PiFinder.catalog_imports.catalog_import_utils import parse_designation + + +@pytest.mark.unit +@pytest.mark.parametrize( + "designation, expected", + [ + ("NGC 40", ("NGC", 40)), + ("NGC 7008", ("NGC", 7008)), + ("NGC7008", ("NGC", 7008)), + ("IC 418", ("IC", 418)), + ("M 27", ("M", 27)), + ("M 76", ("M", 76)), + ("Messier 31", ("M", 31)), + ("Abell 43", ("Abl", 43)), + ("PN A66 80", ("Abl", 80)), + ("Sh 2-176", ("Sh2", 176)), + ("SH 2-216", ("Sh2", 216)), + ("Sharpless 176", ("Sh2", 176)), + ("Cr 24", ("Col", 24)), + ("Collinder 24", ("Col", 24)), + ("Caldwell 14", ("C", 14)), + ("Barnard 33", ("B", 33)), + ("Arp 244", ("Arp", 244)), + ("Lyn 12", ("Lyn", 12)), + ("Har 5", ("Har", 5)), + ("Ta2 21", ("Ta2", 21)), + ], +) +def test_recognized_designations(designation, expected): + assert parse_designation(designation) == expected + + +@pytest.mark.unit +def test_one_letter_prefixes_need_a_source_alias(): + # "A 58" and "B 53" are the double stars of Aitken and van den Bos in + # double-star sources, so only a source that means Abell adds "a". + assert parse_designation("A 58") is None + assert parse_designation("B 53") is None + assert parse_designation("A 43", extra_aliases={"a": "Abl"}) == ("Abl", 43) + + +@pytest.mark.unit +@pytest.mark.parametrize( + "designation", + [ + # Minkowski planetary nebulae. Stripping the hyphen would read these + # as Messier 11, 29, 32 and 92. + "M 1-1", + "M 2-9", + "M 3-2", + "M 1-92", + # Haro planetary nebulae, not Herschel 400 entries. + "H 1-1", + "H 3-29", + # Two halves of one object; expanding it is the loader's job, because + # the trailing digits replace the tail of the first number. + "NGC 650-1", + # Designation families PiFinder has no catalog for. + "K 2- 1", + "He 2-47", + "Vy 2-2", + "Hu 1-2", + "Wray 16-93", + "IRAS 06518-1041", + # Designations that end in digits that are not a sequence. + "WDS 00001+7508", + "PK 120+09.1", + # Nothing to key on. + "40", + "", + "Andromeda", + ], +) +def test_rejected_designations(designation): + assert parse_designation(designation) is None