diff --git a/docs/usage/examples.rst b/docs/usage/examples.rst index 711216ab..8d4f63df 100644 --- a/docs/usage/examples.rst +++ b/docs/usage/examples.rst @@ -223,6 +223,25 @@ the Signature object. - publish: /tmp/idp.xml - stats +If the PKCS#11 library exposes several initialized tokens, select one explicitly +by its token label and, when labels are duplicated, its serial number. For +example, a Luna HA client configured by ``/etc/Chrystoki.conf`` can use: + +.. code-block:: yaml + + - sign: + key: pkcs11:///usr/lib/libCryptoki2_64.so/sctest2 + token_label: sc_ha + token_serial: "11429933786539" + cert: /etc/credentials/sc-test-md-signer.crt + +``token_serial`` is optional when ``token_label`` uniquely identifies the +token. Alternatively, use ``slot_id`` or the historical ``:slot`` URI suffix, +for example ``pkcs11:///usr/lib/libCryptoki2_64.so:17/sctest2``. Do not combine +a slot selector with ``token_label`` or ``token_serial``. The token label selects +the HSM token; the final URI component (``sctest2`` above) selects the key object +within that token. + Running this example requires some preparation. Run the 'p11setup.sh' script in the examples directory. This results in a SoftHSM token being setup with the PIN 'secret1' and SO_PIN 'secret2'. Now run pyFF (assuming you are using a unix-like environment). @@ -231,4 +250,3 @@ you are using a unix-like environment). # env PYKCS11PIN=secret1 SOFTHSM_CONF=softhsm.conf pyff --loglevel=DEBUG p11.fd - diff --git a/pyproject.toml b/pyproject.toml index 13f1396b..b0dd08b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,16 +19,16 @@ dependencies = [ "accept-types>=0.4.1", "apscheduler==3.6.3", "cachetools>=5.5.2", - "eval-type-backport>=0.2.2 ; python_full_version == '3.9.*'", + "cryptography>=42", "gunicorn==25.0.3 ; python_full_version >= '3.10'", "gunicorn>=23.0.0", - "lxml>=4.1.1", "mako>=1.3.10", "pyconfig>=3.2.3,<3.3.0", "pydantic>=2.8", "pyramid>=2.0.2", "pytz>=2025.2", - "pyxmlsecurity>=1.0.0", + "pyuppsala>=0.11.0", + "pybergshamra>=0.9.0", "pyyaml>=3.10", "redis>=5.3.0", "redis-collections>=0.13.0", diff --git a/src/pyff/api.py b/src/pyff/api.py index 342c4877..4eb751cd 100644 --- a/src/pyff/api.py +++ b/src/pyff/api.py @@ -11,11 +11,11 @@ import requests from accept_types import AcceptableType from cachetools import TTLCache -from lxml import etree from pyramid.config import Configurator from pyramid.events import NewRequest from pyramid.request import Request from pyramid.response import Response +from pyuppsala import etree from pyff import __version__ from pyff.constants import config diff --git a/src/pyff/builtins.py b/src/pyff/builtins.py index 89b623fb..6d59512d 100644 --- a/src/pyff/builtins.py +++ b/src/pyff/builtins.py @@ -10,6 +10,7 @@ import os import re import sys +import threading import traceback from copy import deepcopy from datetime import datetime @@ -17,9 +18,9 @@ from typing import Optional from urllib.parse import quote_plus, urlparse -import xmlsec -from lxml import etree -from lxml.etree import DocumentInvalid +import pybergshamra +from pyuppsala import etree +from pyuppsala.etree import DocumentInvalid from str2bool import str2bool from pyff.constants import NS @@ -43,10 +44,13 @@ sort_entities, ) from pyff.utils import ( + cert_dict, + cert_info, datetime2iso, dumptree, duration2timedelta, hash_id, + is_document_root, iso2datetime, parse_xml, root, @@ -63,6 +67,14 @@ FILESPEC_REGEX = r'([^ \t\n\r\f\v]+)\s+as\s+([^ \t\n\r\f\v]+)' log = get_log(__name__) +# PKCS#11 providers own the dynamically loaded vendor module. Keep them alive +# for the worker process lifetime: some modules (including Luna) start native +# background threads that outlive a provider object, so unloading and loading +# the module again can leave those threads executing unmapped code. Sessions +# remain short-lived and are opened separately for each signing operation. +_pkcs11_provider_cache = {} +_pkcs11_provider_cache_lock = threading.Lock() + @pipe def dump(req: Plumbing.Request, *opts): @@ -431,7 +443,7 @@ def info(req: Plumbing.Request, *opts): if req.t is None: raise PipeException("Your pipeline is missing a select statement.") - for e in req.t.xpath("//md:EntityDescriptor", namespaces=NS, smart_strings=False): + for e in iter_entities(req.t): print(e.get('entityID')) return req.t @@ -707,7 +719,16 @@ def load(req: Plumbing.Request, *opts): req.md.rm.add_child(url, child_opts) log.debug("Refreshing all resources") - req.md.rm.reload(fail_on_error=bool(_opts['fail_on_error'])) + req.md.rm.reload( + fail_on_error=bool(_opts['fail_on_error']), + max_workers=int(_opts['max_workers']), + ) + + # Resource trees are only an intermediate representation for loading and + # watcher callbacks. The store has its own entity copies after reload; keep + # the resource graph/metadata but release large parsed source documents. + for resource in req.md.rm.walk(): + resource.t = None def _select_args(req): @@ -1109,6 +1130,102 @@ def _discojson_sp_attr(req, *opts): return json.dumps(res) +def _parse_pkcs11_key_uri(key_uri: str) -> tuple[str, str, str, Optional[int]]: + """Parse pyFF's historical PKCS#11 key URI syntax.""" + if not key_uri.startswith("pkcs11://"): + raise PipeException(f"Malformed pkcs11 uri: {key_uri}") + + module_and_slot, separator, label_and_query = key_uri[len("pkcs11://"):].rpartition('/') + if not separator or not module_and_slot or not label_and_query: + raise PipeException(f"Malformed pkcs11 uri (missing module path or object label): {key_uri}") + + label, _, query = label_and_query.partition('?') + if not label: + raise PipeException(f"Malformed pkcs11 uri (missing object label): {key_uri}") + + module_path = module_and_slot + uri_slot_id = None + path_without_slot, colon, slot_text = module_and_slot.rpartition(':') + if colon and re.fullmatch(r"-?\d+", slot_text): + try: + uri_slot_id = int(slot_text) + except ValueError as ex: + raise PipeException(f"Invalid PKCS#11 slot ID {slot_text!r}") from ex + if uri_slot_id < 0: + raise PipeException(f"Invalid PKCS#11 slot ID {slot_text!r}") + module_path = path_without_slot + + pin_spec = "env:PYKCS11PIN" + if query: + for param in query.split('&'): + key, _, value = param.partition('=') + if key == 'pin': + pin_spec = value + + return module_path, label, pin_spec, uri_slot_id + + +def _select_pkcs11_provider( + module_path: str, + *, + uri_slot_id: Optional[int] = None, + slot_id=None, + token_label=None, + token_serial=None, +): + """Get a process-cached provider using exactly one token selector.""" + configured_slot_id = None + if slot_id is not None: + if isinstance(slot_id, bool): + raise PipeException(f"Invalid PKCS#11 slot ID {slot_id!r}") + try: + configured_slot_id = int(slot_id) + except (TypeError, ValueError) as ex: + raise PipeException(f"Invalid PKCS#11 slot ID {slot_id!r}") from ex + if configured_slot_id < 0: + raise PipeException(f"Invalid PKCS#11 slot ID {slot_id!r}") + + if uri_slot_id is not None and configured_slot_id is not None and uri_slot_id != configured_slot_id: + raise PipeException("Conflicting PKCS#11 slot IDs in key URI and sign configuration") + + effective_slot_id = configured_slot_id if configured_slot_id is not None else uri_slot_id + if token_serial is not None and token_label is None: + raise PipeException("PKCS#11 token_serial requires token_label") + if effective_slot_id is not None and (token_label is not None or token_serial is not None): + raise PipeException("PKCS#11 slot_id cannot be combined with token_label or token_serial") + + normalized_token_label = None if token_label is None else str(token_label) + normalized_token_serial = None if token_serial is None else str(token_serial) + cache_key = ( + os.path.realpath(module_path), + effective_slot_id, + normalized_token_label, + normalized_token_serial, + ) + + # Hold the lock through construction. functools.lru_cache and a + # check-then-create sequence can both invoke the constructor more than once + # during concurrent first access, which is precisely what vendor modules + # with process-global initialization cannot safely tolerate. + with _pkcs11_provider_cache_lock: + if cache_key in _pkcs11_provider_cache: + return _pkcs11_provider_cache[cache_key] + + if normalized_token_label is not None: + provider = pybergshamra.Pkcs11Provider.with_token( + module_path, + normalized_token_label, + token_serial=normalized_token_serial, + ) + elif effective_slot_id is not None: + provider = pybergshamra.Pkcs11Provider.with_slot_id(module_path, effective_slot_id) + else: + provider = pybergshamra.Pkcs11Provider(module_path) + + _pkcs11_provider_cache[cache_key] = provider + return provider + + @pipe def sign(req: Plumbing.Request, *_opts): """ @@ -1124,11 +1241,15 @@ def sign(req: Plumbing.Request, *_opts): The 'cert' argument may be empty in which case the cert is looked up using the PKCS#11 token, or may point to a file containing a PEM-encoded X.509 certificate. + When several initialized tokens are visible, select one with ``token_label`` and optional ``token_serial``, + or with ``slot_id``. Token selection requires pybergshamra 0.8.1 or newer. The slot may alternatively be + supplied using the historical ``:slot`` suffix in the key URI. Slot and token selectors are mutually exclusive. + **PKCS11 URIs** A pkcs11 URI has the form - .. code-block:: xml + .. code-block:: text pkcs11://[:slot]/[?pin=] @@ -1141,9 +1262,13 @@ def sign(req: Plumbing.Request, *_opts): .. code-block:: yaml - sign: - key: pkcs11:///usr/lib/libsofthsm.so/signer + key: pkcs11:///usr/lib/libCryptoki2_64.so/sctest2 + token_label: sc_ha + token_serial: "11429933786539" # optional when the label is unique + cert: /etc/credentials/sc-test-md-signer.crt - This would sign the document using the key with label 'signer' in slot 0 of the /usr/lib/libsofthsm.so module. + This signs with the key-object label ``sctest2`` on the token labelled ``sc_ha``. The PKCS#11 client library + still reads vendor connectivity and HA settings from its normal configuration, such as ``/etc/Chrystoki.conf``. Note that you may need to run pyff with env PYKCS11PIN= .... for this to work. Consult the documentation of your PKCS#11 module to find out about any other configuration you may need. @@ -1167,16 +1292,87 @@ def sign(req: Plumbing.Request, *_opts): if key_file is None: raise PipeException("Missing key argument for sign pipe") + if not isinstance(key_file, str): + raise PipeException("The sign pipe key argument must be a string") if cert_file is None: log.info("Attempting to extract certificate from token...") - opts = dict() + # Build the signing context. The reference id (if any) comes from the @ID + # attribute on the document root; pybergshamra already treats "ID" as a + # default id attribute, so we must NOT call ctx.add_id_attr("ID"). relt = root(req.t) idattr = relt.get('ID') - if idattr: - opts['reference_uri'] = f'#{idattr}' - xmlsec.sign(req.t, key_file, cert_file, **opts) + + # Read the certificate (if provided) as PEM text to embed in X509Data. + cert_pem = None + if cert_file is not None: + with open(cert_file) as fd: + cert_pem = fd.read() + + mgr = pybergshamra.KeysManager() + ctx = pybergshamra.DsigContext(mgr) + + is_pkcs11_key = key_file.startswith("pkcs11:") + if is_pkcs11_key: + # HSM signing via PKCS#11. The URI form is: + # pkcs11://[:slot]/