Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,15 @@ jobs:
with:
python-version: "3.11"

- name: Install Java and Node
run: sudo apt update && sudo apt install -y default-jdk nodejs npm git curl
# Nextflow 23.10 supports Java up to 21
- name: Set up Java 21
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: "21"

- name: Install Node
run: sudo apt update && sudo apt install -y nodejs npm git curl

- name: Install vcf-validator and vcf-assembly-checker
run: |
Expand Down
47 changes: 47 additions & 0 deletions eva_sub_cli/etc/eva_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,11 @@
"type": "string",
"description": "The path to the reference in fasta format that was used to create the VCF. (This is a local path)"
},
"referenceSequenceCollection": {
"type": "string",
"description": "GA4GH sequence collection top level digest of the reference fasta used for validation. This is calculated by eva-sub-cli and should not be provided.",
"pattern": "^[A-Za-z0-9_-]{32}$"
},
"assemblyReport": {
"type": "string",
"description": "The path to the assembly report associated with the assembly as found in NCBI assemblies. (This is a local path and only applicable for GCAs)"
Expand Down Expand Up @@ -428,6 +433,48 @@
}
}
}
},
"sequenceCollections": {
"type": "array",
"description": "GA4GH sequence collections of the reference fasta files used for validation. This is calculated by eva-sub-cli and should not be provided.",
"items": {
"type": "object",
"required": [
"digest",
"names",
"lengths",
"sequences"
],
"properties": {
"digest": {
"type": "string",
"description": "Top level digest of the sequence collection",
"pattern": "^[A-Za-z0-9_-]{32}$"
},
"names": {
"type": "array",
"description": "Names of the sequences",
"items": {
"type": "string"
}
},
"lengths": {
"type": "array",
"description": "Lengths of the sequences",
"items": {
"type": "integer"
}
},
"sequences": {
"type": "array",
"description": "Refget digests of the sequences",
"items": {
"type": "string",
"pattern": "^SQ\\.[A-Za-z0-9_-]{32}$"
}
}
}
}
}
},
"definitions": {
Expand Down
27 changes: 27 additions & 0 deletions eva_sub_cli/executables/generate_sequence_collection.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import argparse
import json

from ebi_eva_common_pyutils.logger import logging_config

from eva_sub_cli.seqcol import sequence_collection_from_fasta

logger = logging_config.get_logger(__name__)


def main():
arg_parser = argparse.ArgumentParser(
description='Calculate the GA4GH sequence collection (and its digest) of a FASTA file.')
arg_parser.add_argument('--input_fasta', required=True, dest='input_fasta',
help='Fasta file for which the sequence collection will be calculated')
arg_parser.add_argument('--output_json', required=True, dest='output_json',
help='Path to the JSON file where the sequence collection will be written')
args = arg_parser.parse_args()
logging_config.add_stdout_handler()
sequence_collection = sequence_collection_from_fasta(args.input_fasta)
logger.info(f'Sequence collection digest for {args.input_fasta}: {sequence_collection["digest"]}')
with open(args.output_json, 'w') as open_json:
json.dump(sequence_collection, open_json)


if __name__ == "__main__":
main()
8 changes: 8 additions & 0 deletions eva_sub_cli/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,14 @@ def set_analyses(self, analysis_list):
assert isinstance(analysis_list, list)
self.content['analysis'] = analysis_list

@property
def sequence_collections(self):
return self.content.get('sequenceCollections', [])

def set_sequence_collections(self, sequence_collection_list):
assert isinstance(sequence_collection_list, list)
self.content['sequenceCollections'] = sequence_collection_list

@cached_property
def samples_per_analysis(self):
"""Returns mapping of analysis alias to sample names, based on metadata."""
Expand Down
28 changes: 28 additions & 0 deletions eva_sub_cli/nextflow/validation.nf
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ params.python_scripts = [
"samples_checker": "samples_checker.py",
"evidence_type_checker": "evidence_type_checker.py",
"fasta_checker": "check_fasta_insdc.py",
"sequence_collection": "generate_sequence_collection.py",
"xlsx2json": "xlsx2json.py",
"semantic_checker": "check_metadata_semantics.py",
"trim_down": "trim_down.py",
Expand Down Expand Up @@ -85,6 +86,8 @@ workflow {
file(joinBasePath(row.fasta)),
file(joinBasePath(row.report))
)}
// Sequence collection is calculated on the original (not trimmed down) fasta files
generate_sequence_collection(vcf_and_ref_ch.map{row -> row[1]}.unique())
if (params.shallow_validation) {
// create a smaller vcf and fasta then replace the channel
trim_down_vcf(vcf_and_ref_ch)
Expand Down Expand Up @@ -253,6 +256,31 @@ process collect_file_size_and_md5 {
}


/*
* Calculate the GA4GH sequence collection of the reference fasta
*/
process generate_sequence_collection {

label 'med_time', 'small_mem'

publishDir output_dir,
overwrite: true,
mode: "copy"

input:
path(fasta)

output:
path "${fasta}_seqcol.json", emit: sequence_collection_json
path "${fasta}_seqcol.log", emit: sequence_collection_log

script:
"""
$params.python_scripts.sequence_collection --input_fasta $fasta --output_json ${fasta}_seqcol.json > ${fasta}_seqcol.log 2>&1
"""
}


process convert_xlsx_2_json {

label 'short_time', 'small_mem'
Expand Down
57 changes: 57 additions & 0 deletions eva_sub_cli/seqcol.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
"""
Computation of GA4GH sequence collections (https://ga4gh.github.io/refget/seqcols/) from a FASTA file.
"""
import base64
import hashlib
import json

from eva_sub_cli.file_utils import fasta_iter

# Attributes included in the level 2 representation of the sequence collection
SEQCOL_ATTRIBUTES = ('names', 'lengths', 'sequences')
# Attributes contributing to the top level (level 0) digest
INHERENT_ATTRIBUTES = ('names', 'sequences')


def sha512t24u_digest(data):
"""GA4GH digest: base64url encoding of the first 24 bytes of the SHA-512 hash of the data."""
if isinstance(data, str):
data = data.encode('utf-8')
return base64.urlsafe_b64encode(hashlib.sha512(data).digest()[:24]).decode('ascii')


def canonical_json(obj):
"""
Serialise to canonical JSON (RFC-8785). This implementation is sufficient for objects and arrays containing
strings and integers, which is all that the sequence collection attributes use.
"""
return json.dumps(obj, separators=(',', ':'), sort_keys=True, ensure_ascii=False, allow_nan=False).encode('utf-8')


def refget_sequence_digest(sequence):
"""Refget v2 identifier of a sequence."""
return 'SQ.' + sha512t24u_digest(sequence.upper())


def level1_digests(collection):
"""Digest of each attribute of a level 2 sequence collection."""
return {attribute: sha512t24u_digest(canonical_json(collection[attribute])) for attribute in SEQCOL_ATTRIBUTES}


def level0_digest(level1):
"""Top level digest of a sequence collection, computed from the level 1 digests of the inherent attributes."""
return sha512t24u_digest(canonical_json({attribute: level1[attribute] for attribute in INHERENT_ATTRIBUTES}))


def sequence_collection_from_fasta(input_fasta):
"""
Calculate the sequence collection of a FASTA file.
:param input_fasta: path to the (optionally gzipped) FASTA file
:returns: dict with the level 0 digest and the level 2 attributes (names, lengths, sequences)
"""
collection = {attribute: [] for attribute in SEQCOL_ATTRIBUTES}
for header, sequence in fasta_iter(input_fasta):
collection['names'].append(header.split()[0])
collection['lengths'].append(len(sequence))
collection['sequences'].append(refget_sequence_digest(sequence))
return {'digest': level0_digest(level1_digests(collection)), **collection}
49 changes: 49 additions & 0 deletions eva_sub_cli/validators/validator.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env python
import csv
import datetime
import json
import logging
import os
import shutil
Expand Down Expand Up @@ -527,6 +528,7 @@ def _collect_metadata_results(self):
self._convert_biovalidator_validation_to_spreadsheet()
self._write_spreadsheet_validation_results()
file_info_file = self._collect_file_info_to_metadata()
self._collect_sequence_collections_to_metadata()
self._add_schema_to_metadata()

if not any([metadata_validation_file, metadata_semantic_file, file_info_file,
Expand Down Expand Up @@ -706,6 +708,53 @@ def _collect_file_info_to_metadata(self):
self.results[METADATA_CHECK]['json_errors'] = errors
return md5sum_file

def _load_sequence_collections(self):
"""Returns the sequence collections calculated for each fasta file, keyed by fasta file name."""
sequence_collections = {}
for fasta_file in self.fasta_files:
fasta_name = os.path.basename(fasta_file)
seqcol_file = resolve_single_file_path(
os.path.join(self.output_dir, 'other_validations', f'{fasta_name}_seqcol.json'))
if not seqcol_file:
self.warning(f'Cannot locate the sequence collection for {fasta_name}. The process might have failed.')
continue
try:
with open(seqcol_file) as open_json:
sequence_collections[fasta_name] = json.load(open_json)
except ValueError as e:
self.warning(f'Cannot parse the sequence collection in {seqcol_file}: {e}')
return sequence_collections

def _collect_sequence_collections_to_metadata(self):
"""
Attach the sequence collection digest of the reference fasta to each analysis and add the sequence
collections to the metadata.
This is not used for validation so failures are only logged.
"""
if not self.metadata_json_post_validation:
return
sequence_collections = self._load_sequence_collections()
metadata = EvaMetadataJson(self.metadata_json_post_validation)
try:
collections_per_digest = {}
for analysis in metadata.analyses:
if not analysis.get('referenceFasta'):
continue
sequence_collection = sequence_collections.get(os.path.basename(analysis['referenceFasta']))
if not sequence_collection:
self.warning(f'No sequence collection found for analysis {analysis.get("analysisAlias")}')
continue
analysis['referenceSequenceCollection'] = sequence_collection['digest']
collections_per_digest[sequence_collection['digest']] = sequence_collection
if not collections_per_digest:
return
metadata.set_analyses(metadata.analyses)
metadata.set_sequence_collections(list(collections_per_digest.values()))
except Exception as e:
self.warning('Error while adding sequence collections to the metadata json: ' + str(e))
return
metadata.write(self.metadata_json_post_validation)

def _add_schema_to_metadata(self):
if self.metadata_json_post_validation:
metadata = EvaMetadataJson(self.metadata_json_post_validation)
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ classifiers = [
[project.scripts]
'eva-sub-cli.py'='eva_sub_cli.executables.cli:main'
'check_fasta_insdc.py'='eva_sub_cli.executables.check_fasta_insdc:main'
'generate_sequence_collection.py'='eva_sub_cli.executables.generate_sequence_collection:main'
'check_metadata_semantics.py'='eva_sub_cli.executables.check_metadata_semantics:main'
'samples_checker.py'='eva_sub_cli.executables.samples_checker:main'
'evidence_type_checker.py'='eva_sub_cli.executables.evidence_type_checker:main'
Expand Down
6 changes: 6 additions & 0 deletions tests/resources/seqcol/base.fa
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
>chrX
TTGGGGAA
>chr1
GGAA
>chr2
GCGC
6 changes: 6 additions & 0 deletions tests/resources/seqcol/different_names.fa
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
>X
TTGGGGAA
>1
GGAA
>2
GCGC
6 changes: 6 additions & 0 deletions tests/resources/seqcol/different_order.fa
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
>chr1
GGAA
>chr2
GCGC
>chrX
TTGGGGAA
Loading
Loading