diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 958b87e..9f7353e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -4,7 +4,6 @@ on: push: tags: - 'v*.*.*' - - 'test*' workflow_dispatch: permissions: diff --git a/pyproject.toml b/pyproject.toml index 7fa2574..8a8e2c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pmotools" -version = "v1.1.0" +version = "v1.2.0" description = "Tools for building and analyzing PMO files" readme = "README.md" authors = [ diff --git a/src/pmotools/pmo_builder/metatable_to_pmo.py b/src/pmotools/pmo_builder/metatable_to_pmo.py index 4162c22..9d1d2ab 100644 --- a/src/pmotools/pmo_builder/metatable_to_pmo.py +++ b/src/pmotools/pmo_builder/metatable_to_pmo.py @@ -157,7 +157,7 @@ def library_sample_info_table_to_pmo( library_prep_plate_position_col, meta_json, copy_contents, - "specimen_name", + "library_sample_name", "library_prep_plate_info", ) meta_json = add_parasite_density_info( @@ -166,7 +166,7 @@ def library_sample_info_table_to_pmo( meta_json, copy_contents, "library_sample_name", - entry_name="parasite_density_info", + entry_name="qpcr_parasite_density_info", ) # listify columns that contain values that could be list, are delimited by the argument list_values_library_values_delimiter primitives = (int, float, str, bool, complex) @@ -212,7 +212,7 @@ def specimen_info_table_to_pmo( has_travel_out_six_month_col: str = None, host_age_col: str = None, host_sex_col: str = None, - host_subject_id: str = None, + host_subject_name_col: str = None, lat_lon_col: str = None, parasite_density_col: str = None, parasite_density_method_col: str = None, @@ -281,8 +281,8 @@ def specimen_info_table_to_pmo( :type host_age_col: str, optional :param host_sex_col: if the specimen is from a person, the sex of that person :type host_sex_col: str, optional - :param host_subject_id: ID for the individual a specimen was collected from - :type host_subject_id: str, optional + :param host_subject_name_col: identifier for the individual a specimen was collected from + :type host_subject_name_col: str, optional :param lat_lon_col: latitude and longitude of the collection site :type lat_lon_col: str, optional :param parasite_density_col: the parasite density in parasites per microliter @@ -353,7 +353,7 @@ def specimen_info_table_to_pmo( geo_admin3_col: "geo_admin3", host_age_col: "host_age", host_sex_col: "host_sex", - host_subject_id: "host_subject_id", + host_subject_name_col: "host_subject_name", lat_lon_col: "lat_lon", specimen_accession_col: "specimen_accession", specimen_type_col: "specimen_type", @@ -392,7 +392,7 @@ def specimen_info_table_to_pmo( geo_admin3_col, host_age_col, host_sex_col, - host_subject_id, + host_subject_name_col, lat_lon_col, specimen_accession_col, specimen_type_col, @@ -487,6 +487,14 @@ def check_columns_exist(df, columns): ) +def _is_blank(value): + if value is None: + return True + if isinstance(value, str): + return not value.strip() + return bool(pd.isna(value)) + + def add_plate_info( plate_col_col, plate_name_col, @@ -494,7 +502,7 @@ def add_plate_info( plate_position_col, meta_json, df, - specimen_name_col, + match_col, entry_name="plate_info", ): if all( @@ -516,38 +524,39 @@ def add_plate_info( plate_row_col = "plate_row" plate_col_col = "plate_col" - try: - df[plate_row_col] = ( - df[plate_position_col].str.extract(r"(?i)^([A-H])")[0].str.upper() - ) - df[plate_col_col] = ( - df[plate_position_col] - .str.extract(r"(?i)^[A-H]0*([1-9]|1[0-2])$")[0] - .astype(int) - ) - except (AttributeError, ValueError, IndexError, KeyError) as e: + positions = df[plate_position_col] + blank = positions.map(_is_blank) + parsed = ( + positions[~blank] + .astype(str) + .str.strip() + .str.extract(r"(?i)^([A-H])0*([1-9]|1[0-2])$") + ) + if parsed.isna().any().any(): raise ValueError( f"Values in '{plate_position_col}' must start with a single letter A-H/a-h followed by number 1-12." - ) from e + ) + df[plate_row_col] = None + df.loc[~blank, plate_row_col] = parsed[0].str.upper() + df[plate_col_col] = pd.Series(pd.NA, index=df.index, dtype="Int64") + df.loc[~blank, plate_col_col] = parsed[1].astype(int) for row in meta_json: - content_row = df[df[specimen_name_col] == row[specimen_name_col]] + content_row = df[df[match_col] == row[match_col]] plate_name_val = content_row[plate_name_col].iloc[0] if plate_name_col else None - plate_row_val = ( - content_row[plate_row_col].iloc[0].upper() if plate_row_col else None - ) + plate_row_val = content_row[plate_row_col].iloc[0] if plate_row_col else None plate_col_val = content_row[plate_col_col].iloc[0] if plate_col_col else None - if plate_col_val is not None and not pd.isna(plate_col_val): + if not _is_blank(plate_col_val): try: plate_col_val = int(plate_col_val) except (TypeError, ValueError): plate_col_val = plate_col_val plate_info = {} - if plate_name_val: + if not _is_blank(plate_name_val): plate_info["plate_name"] = plate_name_val - if plate_row_val: - plate_info["plate_row"] = plate_row_val - if plate_col_val is not None and not pd.isna(plate_col_val): + if not _is_blank(plate_row_val): + plate_info["plate_row"] = str(plate_row_val).strip().upper() + if not _is_blank(plate_col_val): plate_info["plate_col"] = plate_col_val if plate_info: @@ -560,7 +569,7 @@ def add_parasite_density_info( parasite_density_method_col, meta_json, df, - specimen_name_col, + match_col, entry_name, ): density_method_pairs = [] @@ -608,7 +617,7 @@ def add_parasite_density_info( # Add parasite density info to meta_json for row in meta_json: - content_row = df[df[specimen_name_col] == row[specimen_name_col]] + content_row = df[df[match_col] == row[match_col]] density_infos = [] for density_col, method_col in density_method_pairs: density_val = content_row[density_col].iloc[0] if density_col else None diff --git a/src/pmotools/pmo_builder/mhap_table_to_pmo.py b/src/pmotools/pmo_builder/mhap_table_to_pmo.py index be0ae34..32f0916 100644 --- a/src/pmotools/pmo_builder/mhap_table_to_pmo.py +++ b/src/pmotools/pmo_builder/mhap_table_to_pmo.py @@ -26,6 +26,13 @@ def mhap_table_to_pmo( masking_delim: str = ",", microhaplotype_name_col: str | None = None, pseudocigar_col: str | None = None, + pseudocigar_chrom_col: str | None = None, + pseudocigar_start_col: str | None = None, + pseudocigar_end_col: str | None = None, + pseudocigar_ref_seq_col: str | None = None, + pseudocigar_strand_col: str | None = None, + pseudocigar_genome_id: int | None = None, + pseudocigar_generation_description_col: str | None = None, quality_col: str | None = None, additional_representative_mhap_cols: list | None = None, additional_mhap_detected_cols: list | None = None, @@ -73,6 +80,20 @@ def mhap_table_to_pmo( :type microhaplotype_name_col: str, optional :param pseudocigar_col: the name of the column containing a pseudocigar for the microhaplotype :type pseudocigar_col: str, optional + :param pseudocigar_chrom_col: column for the pseudocigar ref_loc chromosome; defaults to chrom_col. chrom_col can only be set together with start_col and end_col, so use pseudocigar_chrom_col when there is no full microhaplotype location + :type pseudocigar_chrom_col: str, optional + :param pseudocigar_start_col: column for the pseudocigar ref_loc start (required if pseudocigar_col is set) + :type pseudocigar_start_col: str, optional + :param pseudocigar_end_col: column for the pseudocigar ref_loc end (required if pseudocigar_col is set) + :type pseudocigar_end_col: str, optional + :param pseudocigar_ref_seq_col: optional column for the pseudocigar ref_loc reference sequence + :type pseudocigar_ref_seq_col: str, optional + :param pseudocigar_strand_col: optional column for the pseudocigar ref_loc strand + :type pseudocigar_strand_col: str, optional + :param pseudocigar_genome_id: genome id for the pseudocigar ref_loc; defaults to genome_id + :type pseudocigar_genome_id: int, optional + :param pseudocigar_generation_description_col: optional column describing how the pseudocigar was generated + :type pseudocigar_generation_description_col: str, optional :param quality_col: the name of the column containing the ANSI FASTQ per-base quality score for this sequence :type quality_col: str, optional :param additional_representative_mhap_cols: additional columns to add to the representative microhaplotypes table @@ -100,6 +121,13 @@ def mhap_table_to_pmo( masking_delim=masking_delim, microhaplotype_name_col=microhaplotype_name_col, pseudocigar_col=pseudocigar_col, + pseudocigar_chrom_col=pseudocigar_chrom_col, + pseudocigar_start_col=pseudocigar_start_col, + pseudocigar_end_col=pseudocigar_end_col, + pseudocigar_ref_seq_col=pseudocigar_ref_seq_col, + pseudocigar_strand_col=pseudocigar_strand_col, + pseudocigar_genome_id=pseudocigar_genome_id, + pseudocigar_generation_description_col=pseudocigar_generation_description_col, quality_col=quality_col, additional_representative_mhap_cols=additional_representative_mhap_cols, ) @@ -160,6 +188,13 @@ def create_representative_microhaplotype_dict( masking_delim: str = ",", microhaplotype_name_col: str | None = None, pseudocigar_col: str | None = None, + pseudocigar_chrom_col: str | None = None, + pseudocigar_start_col: str | None = None, + pseudocigar_end_col: str | None = None, + pseudocigar_ref_seq_col: str | None = None, + pseudocigar_strand_col: str | None = None, + pseudocigar_genome_id: int | None = None, + pseudocigar_generation_description_col: str | None = None, quality_col: str | None = None, additional_representative_mhap_cols: list[str] | None = None, ): @@ -198,6 +233,20 @@ def create_representative_microhaplotype_dict( :type microhaplotype_name_col: str, optional :param pseudocigar_col: the name of the column containing a pseudocigar for the microhaplotype :type pseudocigar_col: str, optional + :param pseudocigar_chrom_col: column for the pseudocigar ref_loc chromosome; defaults to chrom_col. chrom_col can only be set together with start_col and end_col, so use pseudocigar_chrom_col when there is no full microhaplotype location + :type pseudocigar_chrom_col: str, optional + :param pseudocigar_start_col: column for the pseudocigar ref_loc start (required if pseudocigar_col is set) + :type pseudocigar_start_col: str, optional + :param pseudocigar_end_col: column for the pseudocigar ref_loc end (required if pseudocigar_col is set) + :type pseudocigar_end_col: str, optional + :param pseudocigar_ref_seq_col: optional column for the pseudocigar ref_loc reference sequence + :type pseudocigar_ref_seq_col: str, optional + :param pseudocigar_strand_col: optional column for the pseudocigar ref_loc strand + :type pseudocigar_strand_col: str, optional + :param pseudocigar_genome_id: genome id for the pseudocigar ref_loc; defaults to genome_id + :type pseudocigar_genome_id: int, optional + :param pseudocigar_generation_description_col: optional column describing how the pseudocigar was generated + :type pseudocigar_generation_description_col: str, optional :param quality_col: the name of the column containing the ANSI FASTQ per-base quality score for this sequence :type quality_col: str, optional :param additional_representative_mhap_cols: additional columns to add to the representative microhaplotypes table @@ -211,6 +260,24 @@ def create_representative_microhaplotype_dict( microhaplotype_table, additional_representative_mhap_cols ) + # the pseudocigar ref_loc shares the chromosome / genome with the + # microhaplotype location by default, but each may use its own column + pseudocigar_chrom_col = ( + pseudocigar_chrom_col if pseudocigar_chrom_col else chrom_col + ) + ps_genome_id = ( + pseudocigar_genome_id if pseudocigar_genome_id is not None else genome_id + ) + if pseudocigar_col and not ( + pseudocigar_chrom_col and pseudocigar_start_col and pseudocigar_end_col + ): + raise ValueError( + "pseudocigar_col is set, so a Pseudocigar ref_loc must be " + "constructable: set a chromosome (pseudocigar_chrom_col, or " + "chrom_col together with start_col and end_col), " + "pseudocigar_start_col, and pseudocigar_end_col." + ) + def get_if_present(row, col): return row[col] if col and pd.notna(row[col]) else None @@ -263,6 +330,12 @@ def warn_if_duplicated_seqs(df, target_col, seq_col): alt_annotations_col, microhaplotype_name_col, pseudocigar_col, + pseudocigar_chrom_col, + pseudocigar_start_col, + pseudocigar_end_col, + pseudocigar_ref_seq_col, + pseudocigar_strand_col, + pseudocigar_generation_description_col, quality_col, ] masking_cols = [ @@ -319,7 +392,34 @@ def warn_if_duplicated_seqs(df, target_col, seq_col): if val := get_if_present(row, microhaplotype_name_col): mhap["microhaplotype_name"] = val if val := get_if_present(row, pseudocigar_col): - mhap["pseudo_cigar"] = val + if not ( + pd.notna(row[pseudocigar_chrom_col]) + and pd.notna(row[pseudocigar_start_col]) + and pd.notna(row[pseudocigar_end_col]) + ): + raise ValueError( + f"pseudocigar present for target {target}, seq " + f"{row[seq_col]} but its ref_loc chrom/start/end " + "is missing" + ) + ref_loc = { + "genome_id": ps_genome_id, + "chrom": row[pseudocigar_chrom_col], + "start": row[pseudocigar_start_col], + "end": row[pseudocigar_end_col], + } + if pseudocigar_ref_seq_col and pd.notna(row[pseudocigar_ref_seq_col]): + ref_loc["ref_seq"] = row[pseudocigar_ref_seq_col] + if pseudocigar_strand_col and pd.notna(row[pseudocigar_strand_col]): + ref_loc["strand"] = row[pseudocigar_strand_col] + pseudocigar = {"pseudocigar_seq": val, "ref_loc": ref_loc} + if pseudocigar_generation_description_col and pd.notna( + row[pseudocigar_generation_description_col] + ): + pseudocigar["pseudocigar_generation_description"] = row[ + pseudocigar_generation_description_col + ] + mhap["pseudocigar"] = pseudocigar if val := get_if_present(row, quality_col): mhap["quality"] = val if additional_representative_mhap_cols: diff --git a/src/pmotools/pmo_builder/panel_information_to_pmo.py b/src/pmotools/pmo_builder/panel_information_to_pmo.py index 949fd8c..da14689 100644 --- a/src/pmotools/pmo_builder/panel_information_to_pmo.py +++ b/src/pmotools/pmo_builder/panel_information_to_pmo.py @@ -289,9 +289,11 @@ def build_target_info_dict( if self.gene_name_col: target_dict["gene_name"] = row[self.gene_name_col] if self.target_attributes_col: - target_dict["target_attributes"] = row[ - self.target_attributes_col - ].split(self.target_attributes_col_delimiter) + target_attributes = row[self.target_attributes_col] + if pd.notna(target_attributes) and str(target_attributes).strip(): + target_dict["target_attributes"] = str(target_attributes).split( + self.target_attributes_col_delimiter + ) if self.additional_target_info_cols: for col in self.additional_target_info_cols: value = row[col] @@ -332,8 +334,8 @@ def build_target_info_dict( fwd_primer_dict["location"] = { "genome_id": genome_id, "chrom": row[chrom_col], - "end": int(row[forward_primers_start_col]), - "start": int(row[forward_primers_end_col]), + "start": int(row[forward_primers_start_col]), + "end": int(row[forward_primers_end_col]), } if strand_col and pd.notna(row[strand_col]): fwd_primer_dict["location"]["strand"] = row[strand_col] diff --git a/src/pmotools/pmo_builder/pmo_updater.py b/src/pmotools/pmo_builder/pmo_updater.py index c0f9305..c47c474 100644 --- a/src/pmotools/pmo_builder/pmo_updater.py +++ b/src/pmotools/pmo_builder/pmo_updater.py @@ -60,28 +60,24 @@ def update_specimen_meta_with_traveler_info( :param replace_current_traveler_info: whether to replace current travel info :return: a reference to the updated PMO """ - required_cols = [ - specimen_name_col, - travel_country_col, - travel_start_col, - travel_end_col, - ] - if bed_net_usage_col is not None: - required_cols.append(bed_net_usage_col) - if geo_admin1_col is not None: - required_cols.append(geo_admin1_col) - if geo_admin2_col is not None: - required_cols.append(geo_admin2_col) - if geo_admin3_col is not None: - required_cols.append(geo_admin3_col) - if lat_lon_col is not None: - required_cols.append(lat_lon_col) - - if not set(required_cols).issubset(traveler_info.columns): - raise Exception( - "missing traveler_info columns: " + ",".join(required_cols), - " columns in table: " + ",".join(traveler_info.columns), - ) + optional_cols = { + "geo_admin1": geo_admin1_col, + "geo_admin2": geo_admin2_col, + "geo_admin3": geo_admin3_col, + "lat_lon": lat_lon_col, + } + PMOUpdater._check_columns( + traveler_info, + [ + specimen_name_col, + travel_country_col, + travel_start_col, + travel_end_col, + bed_net_usage_col, + *optional_cols.values(), + ], + "traveler_info", + ) specimen_names_in_pmo = set(PMOProcessor.get_specimen_names(pmo)) specimen_names_in_traveler_info = set( @@ -95,7 +91,6 @@ def update_specimen_meta_with_traveler_info( raise ValueError( f"Provided traveler info for the following specimens but they are missing from the PMO: {sorted(missing_traveler_specs)}" ) - traveler_info_records = traveler_info[required_cols].to_dict(orient="records") spec_indexs = PMOProcessor.get_index_key_of_specimen_names(pmo) # prep traveler info lists, clear the list if we are replacing or start an empty list to append to if none exist already @@ -109,12 +104,20 @@ def update_specimen_meta_with_traveler_info( "travel_out_six_month" ] = [] - for travel_rec in traveler_info_records: - specimen_name = str(travel_rec[specimen_name_col]) + for _, row in traveler_info.iterrows(): + specimen_name = str(row[specimen_name_col]) + if PMOUpdater._is_blank(row[travel_country_col]): + raise ValueError( + f"Missing required value in column '{travel_country_col}' for specimen '{specimen_name}'" + ) + travel_rec = {"travel_country": str(row[travel_country_col])} # Validate date formats - for date_col in (travel_start_col, travel_end_col): - val = travel_rec[date_col] - if pd.isna(val): + for date_col, key in ( + (travel_start_col, "travel_start_date"), + (travel_end_col, "travel_end_date"), + ): + val = row[date_col] + if PMOUpdater._is_blank(val): raise ValueError( f"Missing required date value in column '{date_col}' for specimen '{specimen_name}'" ) @@ -124,13 +127,396 @@ def update_specimen_meta_with_traveler_info( f"Invalid date format in '{date_col}' for specimen '{specimen_name}': '{val_str}'. " f"Expected YYYY-MM or YYYY-MM-DD" ) - # add in travel_rec - travel_rec.pop(specimen_name_col, None) + travel_rec[key] = val_str + if bed_net_usage_col and not PMOUpdater._is_blank(row[bed_net_usage_col]): + travel_rec["bed_net_usage"] = float(row[bed_net_usage_col]) + for key, col in optional_cols.items(): + if col and not PMOUpdater._is_blank(row[col]): + travel_rec[key] = str(row[col]) pmo["specimen_info"][spec_indexs[specimen_name]][ "travel_out_six_month" ].append(travel_rec) return pmo + @staticmethod + def _is_blank(value) -> bool: + if value is None: + return True + if isinstance(value, str): + return not value.strip() + return bool(pd.isna(value)) + + @staticmethod + def _check_columns(table: pd.DataFrame, cols: list, table_label: str) -> None: + missing = [col for col in cols if col is not None and col not in table.columns] + if missing: + raise ValueError( + f"Missing {table_label} columns: {missing}. Columns in table: {list(table.columns)}" + ) + + @staticmethod + def _genomic_location_from_row( + row, + context: str, + genome_id_col: str | None, + chrom_col: str, + start_col: str, + end_col: str, + strand_col: str | None = None, + ref_seq_col: str | None = None, + alt_seq_col: str | None = None, + ) -> dict: + """ + Build a GenomicLocation dict from a table row. If genome_id_col is None the genome_id defaults to 0. + """ + required = [chrom_col, start_col, end_col] + if genome_id_col is not None: + required.append(genome_id_col) + for col in required: + if PMOUpdater._is_blank(row[col]): + raise ValueError( + f"Missing required value in column '{col}' for {context}" + ) + try: + location = { + "genome_id": int(row[genome_id_col]) if genome_id_col else 0, + "chrom": str(row[chrom_col]), + "start": int(row[start_col]), + "end": int(row[end_col]), + } + except (TypeError, ValueError) as e: + raise ValueError( + f"genome id, start and end must be integers for {context}" + ) from e + for key, col in ( + ("strand", strand_col), + ("ref_seq", ref_seq_col), + ("alt_seq", alt_seq_col), + ): + if col and not PMOUpdater._is_blank(row[col]): + location[key] = str(row[col]) + return location + + @staticmethod + def _get_representative_microhaplotype( + pmo, target_name: str, seq: str, target_indexes: dict + ) -> dict: + if target_name not in target_indexes: + raise ValueError( + f"Target '{target_name}' has no representative microhaplotypes in the PMO" + ) + microhaplotypes = pmo["representative_microhaplotypes"]["targets"][ + target_indexes[target_name] + ]["microhaplotypes"] + for microhaplotype in microhaplotypes: + if microhaplotype["seq"] == seq: + return microhaplotype + raise ValueError( + f"No representative microhaplotype with seq '{seq}' for target '{target_name}'" + ) + + @staticmethod + def update_target_info_with_markers_of_interest( + pmo, + markers_info: pd.DataFrame, + target_name_col: str = "target_name", + chrom_col: str = "chrom", + start_col: str = "start", + end_col: str = "end", + genome_id_col: str | None = None, + strand_col: str | None = None, + ref_seq_col: str | None = None, + alt_seq_col: str | None = None, + associations_col: str | None = None, + associations_delim: str = ",", + replace_current_markers: bool = False, + ): + """ + Update a PMO's target_info with markers of interest, from a table with one row per marker + + :param pmo: the PMO to update, will directly modify this PMO + :param markers_info: the markers of interest table + :param target_name_col: the column containing the name of the target the marker is covered by + :param chrom_col: the column containing the chromosome of the marker + :param start_col: the column containing the start of the marker, 0-based + :param end_col: the column containing the end of the marker, 0-based + :param genome_id_col: (Optional) the column containing the index into targeted_genomes. Default: 0 for every marker + :param strand_col: (Optional) the column containing the strand of the marker + :param ref_seq_col: (Optional) the column containing the reference sequence of the marker + :param alt_seq_col: (Optional) the column containing an alternative sequence of the marker + :param associations_col: (Optional) the column containing associations with the marker, e.g. SP resistance + :param associations_delim: the delimiter between associations. Default: ',' + :param replace_current_markers: whether to replace current markers of interest for the targets in the table + :return: a reference to the updated PMO + """ + PMOUpdater._check_columns( + markers_info, + [ + target_name_col, + chrom_col, + start_col, + end_col, + genome_id_col, + strand_col, + ref_seq_col, + alt_seq_col, + associations_col, + ], + "markers_info", + ) + target_indexes = PMOProcessor.get_index_key_of_target_names(pmo) + target_names_in_table = set(markers_info[target_name_col].astype(str)) + missing_targets = target_names_in_table - set(target_indexes) + if missing_targets: + raise ValueError( + f"Provided markers of interest for the following targets but they are missing from the PMO: {sorted(missing_targets)}" + ) + + for target_name in target_names_in_table: + target = pmo["target_info"][target_indexes[target_name]] + if replace_current_markers or not target.get("markers_of_interest"): + target["markers_of_interest"] = [] + + for _, row in markers_info.iterrows(): + target_name = str(row[target_name_col]) + marker = { + "marker_location": PMOUpdater._genomic_location_from_row( + row, + f"marker of interest for target '{target_name}'", + genome_id_col, + chrom_col, + start_col, + end_col, + strand_col, + ref_seq_col, + alt_seq_col, + ) + } + if associations_col and not PMOUpdater._is_blank(row[associations_col]): + marker["associations"] = [ + association.strip() + for association in str(row[associations_col]).split( + associations_delim + ) + if association.strip() + ] + pmo["target_info"][target_indexes[target_name]][ + "markers_of_interest" + ].append(marker) + return pmo + + @staticmethod + def update_representative_microhaplotypes_with_seq_variants( + pmo, + seq_variants_info: pd.DataFrame, + target_name_col: str = "target_name", + seq_col: str = "seq", + chrom_col: str = "chrom", + start_col: str = "start", + end_col: str = "end", + genome_id_col: str | None = None, + strand_col: str | None = None, + ref_seq_col: str | None = None, + alt_seq_col: str | None = None, + replace_current_seq_variants: bool = False, + ): + """ + Update a PMO's representative microhaplotypes with associated sequence variants, from a table with one row per variant + + :param pmo: the PMO to update, will directly modify this PMO + :param seq_variants_info: the sequence variants table + :param target_name_col: the column containing the target name of the microhaplotype + :param seq_col: the column containing the sequence of the microhaplotype + :param chrom_col: the column containing the chromosome of the variant + :param start_col: the column containing the start of the variant, 0-based + :param end_col: the column containing the end of the variant, 0-based + :param genome_id_col: (Optional) the column containing the index into targeted_genomes. Default: 0 for every variant + :param strand_col: (Optional) the column containing the strand of the variant + :param ref_seq_col: (Optional) the column containing the reference sequence of the variant + :param alt_seq_col: (Optional) the column containing the alternative sequence of the variant + :param replace_current_seq_variants: whether to replace current sequence variants for the microhaplotypes in the table + :return: a reference to the updated PMO + """ + PMOUpdater._check_columns( + seq_variants_info, + [ + target_name_col, + seq_col, + chrom_col, + start_col, + end_col, + genome_id_col, + strand_col, + ref_seq_col, + alt_seq_col, + ], + "seq_variants_info", + ) + target_indexes = ( + PMOProcessor.get_index_key_of_target_in_representative_microhaplotypes(pmo) + ) + microhaplotypes = [ + PMOUpdater._get_representative_microhaplotype( + pmo, str(row[target_name_col]), str(row[seq_col]), target_indexes + ) + for _, row in seq_variants_info.iterrows() + ] + + for microhaplotype in microhaplotypes: + if replace_current_seq_variants or not microhaplotype.get( + "associated_seq_variants" + ): + microhaplotype["associated_seq_variants"] = [] + + for (_, row), microhaplotype in zip( + seq_variants_info.iterrows(), microhaplotypes + ): + microhaplotype["associated_seq_variants"].append( + PMOUpdater._genomic_location_from_row( + row, + f"sequence variant for target '{row[target_name_col]}'", + genome_id_col, + chrom_col, + start_col, + end_col, + strand_col, + ref_seq_col, + alt_seq_col, + ) + ) + return pmo + + @staticmethod + def update_representative_microhaplotypes_with_protein_variants( + pmo, + protein_variants_info: pd.DataFrame, + target_name_col: str = "target_name", + seq_col: str = "seq", + transcript_col: str = "transcript", + protein_start_col: str = "protein_start", + protein_end_col: str = "protein_end", + protein_genome_id_col: str | None = None, + protein_ref_seq_col: str | None = None, + protein_alt_seq_col: str | None = None, + gene_name_col: str | None = None, + alternative_gene_name_col: str | None = None, + codon_chrom_col: str | None = None, + codon_start_col: str | None = None, + codon_end_col: str | None = None, + codon_genome_id_col: str | None = None, + codon_strand_col: str | None = None, + codon_ref_seq_col: str | None = None, + codon_alt_seq_col: str | None = None, + replace_current_protein_variants: bool = False, + ): + """ + Update a PMO's representative microhaplotypes with associated protein variants, from a table with one row per variant + + :param pmo: the PMO to update, will directly modify this PMO + :param protein_variants_info: the protein variants table + :param target_name_col: the column containing the target name of the microhaplotype + :param seq_col: the column containing the sequence of the microhaplotype + :param transcript_col: the column containing the transcript name, used as the chrom of the protein location + :param protein_start_col: the column containing the start of the variant within the protein, 0-based + :param protein_end_col: the column containing the end of the variant within the protein, 0-based + :param protein_genome_id_col: (Optional) the column containing the index into targeted_genomes for the protein location. Default: 0 + :param protein_ref_seq_col: (Optional) the column containing the reference amino acid(s) + :param protein_alt_seq_col: (Optional) the column containing the alternative amino acid(s) + :param gene_name_col: (Optional) the column containing the gene name + :param alternative_gene_name_col: (Optional) the column containing an alternative gene name + :param codon_chrom_col: (Optional) the column containing the chromosome of the codon. Must be set with codon_start_col and codon_end_col + :param codon_start_col: (Optional) the column containing the genomic start of the codon, 0-based + :param codon_end_col: (Optional) the column containing the genomic end of the codon, 0-based + :param codon_genome_id_col: (Optional) the column containing the index into targeted_genomes for the codon location. Default: 0 + :param codon_strand_col: (Optional) the column containing the strand of the codon + :param codon_ref_seq_col: (Optional) the column containing the reference sequence of the codon + :param codon_alt_seq_col: (Optional) the column containing the alternative sequence of the codon + :param replace_current_protein_variants: whether to replace current protein variants for the microhaplotypes in the table + :return: a reference to the updated PMO + """ + codon_location_cols = [codon_chrom_col, codon_start_col, codon_end_col] + if any(codon_location_cols) and not all(codon_location_cols): + raise ValueError( + "If any of codon_chrom_col, codon_start_col or codon_end_col are set, all three must be." + ) + PMOUpdater._check_columns( + protein_variants_info, + [ + target_name_col, + seq_col, + transcript_col, + protein_start_col, + protein_end_col, + protein_genome_id_col, + protein_ref_seq_col, + protein_alt_seq_col, + gene_name_col, + alternative_gene_name_col, + *codon_location_cols, + codon_genome_id_col, + codon_strand_col, + codon_ref_seq_col, + codon_alt_seq_col, + ], + "protein_variants_info", + ) + target_indexes = ( + PMOProcessor.get_index_key_of_target_in_representative_microhaplotypes(pmo) + ) + microhaplotypes = [ + PMOUpdater._get_representative_microhaplotype( + pmo, str(row[target_name_col]), str(row[seq_col]), target_indexes + ) + for _, row in protein_variants_info.iterrows() + ] + + for microhaplotype in microhaplotypes: + if replace_current_protein_variants or not microhaplotype.get( + "associated_protein_variants" + ): + microhaplotype["associated_protein_variants"] = [] + + for (_, row), microhaplotype in zip( + protein_variants_info.iterrows(), microhaplotypes + ): + context = f"protein variant for target '{row[target_name_col]}'" + variant = { + "protein_location": PMOUpdater._genomic_location_from_row( + row, + context, + protein_genome_id_col, + transcript_col, + protein_start_col, + protein_end_col, + ref_seq_col=protein_ref_seq_col, + alt_seq_col=protein_alt_seq_col, + ) + } + for key, col in ( + ("gene_name", gene_name_col), + ("alternative_gene_name", alternative_gene_name_col), + ): + if col and not PMOUpdater._is_blank(row[col]): + variant[key] = str(row[col]) + if codon_chrom_col and not all( + PMOUpdater._is_blank(row[col]) for col in codon_location_cols + ): + variant[ + "codon_genomic_location" + ] = PMOUpdater._genomic_location_from_row( + row, + f"codon of {context}", + codon_genome_id_col, + codon_chrom_col, + codon_start_col, + codon_end_col, + codon_strand_col, + codon_ref_seq_col, + codon_alt_seq_col, + ) + microhaplotype["associated_protein_variants"].append(variant) + return pmo + @staticmethod def merge_dicts_by_key( main_list: list[dict], diff --git a/src/pmotools/pmo_engine/pmo_exporter.py b/src/pmotools/pmo_engine/pmo_exporter.py index 361ebd7..a2a952c 100644 --- a/src/pmotools/pmo_engine/pmo_exporter.py +++ b/src/pmotools/pmo_engine/pmo_exporter.py @@ -121,10 +121,14 @@ def export_specimen_meta_table(pmodata, separator: str = ",") -> pd.DataFrame: for specimen in pmodata["specimen_info"]: export_row = {} for key, value in specimen.items(): - if "project_id" == key: + if "project_id" == key and "project_info" in pmodata: export_row["project_name"] = pmodata["project_info"][value][ "project_name" ] + elif "project_id" == key: + # project_info is optional as of schema v1.1.0; keep the raw id + # rather than failing to resolve a name that isn't available + export_row[key] = value elif PMOExporter._is_primitive(value): export_row[key] = value elif PMOExporter._is_primitive_list(value): @@ -145,10 +149,14 @@ def export_library_sample_meta_table(pmodata, separator: str = ",") -> pd.DataFr for library_sample in pmodata["library_sample_info"]: export_row = {} for key, value in library_sample.items(): - if "sequencing_info_id" == key: + if "sequencing_info_id" == key and "sequencing_info" in pmodata: export_row["sequencing_info_name"] = pmodata["sequencing_info"][ value ]["sequencing_info_name"] + elif "sequencing_info_id" == key: + # sequencing_info is optional as of schema v1.1.0; keep the raw id + # rather than failing to resolve a name that isn't available + export_row[key] = value elif "specimen_id" == key: export_row["specimen_name"] = pmodata["specimen_info"][value][ "specimen_name" @@ -451,6 +459,7 @@ def write_bed_locs(bed_locs: list[BedLoc], fnp, add_header: bool = False): ] ) ) + f.write("\n") for bed_loc in bed_locs: f.write( "\t".join( @@ -540,11 +549,10 @@ def extract_panels_insert_bed_loc( :param sort_output: whether to sort output by genomic location :return: a list of target inserts, with named tuples with fields: chrom, start, end, name, score, strand, ref_seq, extra_info """ - bed_loc_out = {} + bed_loc_out = [] if select_panel_ids is None: select_panel_ids = list(range(len(pmodata["panel_info"]))) for panel_id in select_panel_ids: - bed_loc_out_per_panel = [] for reaction_id in range(len(pmodata["panel_info"][panel_id]["reactions"])): for target_id in pmodata["panel_info"][panel_id]["reactions"][ reaction_id @@ -589,7 +597,7 @@ def extract_panels_insert_bed_loc( if "ref_seq" not in tar["insert_location"] else tar["insert_location"]["ref_seq"] ) - bed_loc_out_per_panel.append( + bed_loc_out.append( BedLoc( tar["insert_location"]["chrom"], tar["insert_location"]["start"], @@ -602,12 +610,8 @@ def extract_panels_insert_bed_loc( extra_info, ) ) - if sort_output: - return sorted( - bed_loc_out_per_panel, - key=lambda bed: (bed.chrom, bed.start, bed.end), - ) - bed_loc_out[panel_id] = bed_loc_out_per_panel + if sort_output: + return sorted(bed_loc_out, key=lambda bed: (bed.chrom, bed.start, bed.end)) return bed_loc_out @staticmethod diff --git a/src/pmotools/pmo_engine/pmo_processor.py b/src/pmotools/pmo_engine/pmo_processor.py index cef95fb..1e38ca5 100644 --- a/src/pmotools/pmo_engine/pmo_processor.py +++ b/src/pmotools/pmo_engine/pmo_processor.py @@ -20,6 +20,10 @@ def get_index_key_of_bioinformatics_run_names(pmodata): :return: a dictionary of indexes keyed by bioinformatics_run_name """ ret = {} + # bioinformatics_run_info is optional as of schema v1.1.0; a PMO without it + # simply has no run-name to index mapping + if "bioinformatics_run_info" not in pmodata: + return ret for idx, bioinformatics_run in enumerate(pmodata["bioinformatics_run_info"]): ret[bioinformatics_run["bioinformatics_run_name"]] = idx return ret @@ -646,8 +650,6 @@ def filter_pmo_by_library_sample_ids(pmodata, library_sample_ids: set[int]): if "targeted_genomes" in pmodata: pmo_out["targeted_genomes"] = copy.deepcopy(pmodata["targeted_genomes"]) - if "read_counts_by_stage" in pmodata: - pmo_out["read_counts_by_stage"] = [] # need to update read_counts_by_stage, library_sample_info, specimen_info, detected_microhaplotypes # specimen_info @@ -703,6 +705,7 @@ def filter_pmo_by_library_sample_ids(pmodata, library_sample_ids: set[int]): pmo_out["detected_microhaplotypes"].append(new_detected_microhaplotypes) # read_counts_by_stage if "read_counts_by_stage" in pmodata: + pmo_out["read_counts_by_stage"] = [] for read_count in pmodata["read_counts_by_stage"]: new_read_count = { "read_counts_by_library_sample_by_stage": [], @@ -725,6 +728,7 @@ def filter_pmo_by_library_sample_ids(pmodata, library_sample_ids: set[int]): ]["library_sample_id"] = library_id_index_key[ sample["library_sample_id"] ] + pmo_out["read_counts_by_stage"].append(new_read_count) return pmo_out @staticmethod @@ -884,11 +888,12 @@ def filter_pmo_by_target_ids(pmodata, target_ids: set[int]): pmo_out["representative_microhaplotypes"]["targets"] ) # update new target_id index - microhap_info["target_id"] = target_info_index_key[ + microhap_info_copy = copy.deepcopy(microhap_info) + microhap_info_copy["target_id"] = target_info_index_key[ microhap_info["target_id"] ] pmo_out["representative_microhaplotypes"]["targets"].append( - copy.deepcopy(microhap_info) + microhap_info_copy ) # representative_microhaplotypes pmo_out["detected_microhaplotypes"] = [] @@ -908,10 +913,11 @@ def filter_pmo_by_target_ids(pmodata, target_ids: set[int]): for target in sample["target_results"]: if target["mhaps_target_id"] in mhaps_target_id_new_key: # update with new mhaps_target_id id - target["mhaps_target_id"] = mhaps_target_id_new_key[ + target_copy = copy.deepcopy(target) + target_copy["mhaps_target_id"] = mhaps_target_id_new_key[ target["mhaps_target_id"] ] - new_sample["target_results"].append(copy.deepcopy(target)) + new_sample["target_results"].append(target_copy) new_detected_microhaplotypes["library_samples"].append(new_sample) pmo_out["detected_microhaplotypes"].append(new_detected_microhaplotypes) @@ -938,11 +944,12 @@ def filter_pmo_by_target_ids(pmodata, target_ids: set[int]): for target in sample["read_counts_for_targets"]: if target["target_id"] in target_ids: # update with new target_id index - target["target_id"] = target_info_index_key[ + target_copy = copy.deepcopy(target) + target_copy["target_id"] = target_info_index_key[ target["target_id"] ] new_samples["read_counts_for_targets"].append( - copy.deepcopy(target) + target_copy ) new_read_counts_by_bioid[ "read_counts_by_library_sample_by_stage" @@ -1073,23 +1080,29 @@ def extract_from_pmo_with_read_filter(pmodata, read_filter: float): pmo_out = { "pmo_header": copy.deepcopy(pmodata["pmo_header"]), "panel_info": copy.deepcopy(pmodata["panel_info"]), - "sequencing_info": copy.deepcopy(pmodata["sequencing_info"]), "target_info": copy.deepcopy(pmodata["target_info"]), "specimen_info": copy.deepcopy(pmodata["specimen_info"]), "library_sample_info": copy.deepcopy(pmodata["library_sample_info"]), - "project_info": copy.deepcopy(pmodata["project_info"]), - "targeted_genomes": copy.deepcopy(pmodata["targeted_genomes"]), "representative_microhaplotypes": copy.deepcopy( pmodata["representative_microhaplotypes"] ), - "bioinformatics_methods_info": copy.deepcopy( - pmodata["bioinformatics_methods_info"] - ), - "bioinformatics_run_info": copy.deepcopy( - pmodata["bioinformatics_run_info"] - ), "detected_microhaplotypes": [], } + # only copy optional sections if they are present (no longer required by the schema) + if "sequencing_info" in pmodata: + pmo_out["sequencing_info"] = copy.deepcopy(pmodata["sequencing_info"]) + if "project_info" in pmodata: + pmo_out["project_info"] = copy.deepcopy(pmodata["project_info"]) + if "targeted_genomes" in pmodata: + pmo_out["targeted_genomes"] = copy.deepcopy(pmodata["targeted_genomes"]) + if "bioinformatics_methods_info" in pmodata: + pmo_out["bioinformatics_methods_info"] = copy.deepcopy( + pmodata["bioinformatics_methods_info"] + ) + if "bioinformatics_run_info" in pmodata: + pmo_out["bioinformatics_run_info"] = copy.deepcopy( + pmodata["bioinformatics_run_info"] + ) # if has optional read_counts_by_stage then add as well # if does contain, @todo consider updating with new counts now that a filter has been applied diff --git a/src/pmotools/pmo_engine/pmo_reader.py b/src/pmotools/pmo_engine/pmo_reader.py index 91a6e46..deb7930 100644 --- a/src/pmotools/pmo_engine/pmo_reader.py +++ b/src/pmotools/pmo_engine/pmo_reader.py @@ -394,10 +394,11 @@ def combine_multiple_pmos(pmos: list[dict]): pmo["bioinformatics_run_info"] ): bioinformatics_run_info_copy = copy.deepcopy(bioinformatics_run_info) + # remap using the run's own methods id, not its position index bioinformatics_run_info_copy[ "bioinformatics_methods_id" ] = bioinformatics_methods_info_old_index_key[pmo_index][ - bioinformatics_run_info_index + bioinformatics_run_info_copy["bioinformatics_methods_id"] ] if "bioinformatics_run_info" not in pmo_out: pmo_out["bioinformatics_run_info"] = [] @@ -501,8 +502,13 @@ def combine_multiple_pmos(pmos: list[dict]): new_mhaps_target_index = len( pmo_out["representative_microhaplotypes"]["targets"] ) + new_rep_target = copy.deepcopy(representative_microhaplotypes) + # remap the new target's target_id to the combined target_info + new_rep_target["target_id"] = target_info_old_index_key[pmo_index][ + new_rep_target["target_id"] + ] pmo_out["representative_microhaplotypes"]["targets"].append( - copy.deepcopy(representative_microhaplotypes) + new_rep_target ) representative_microhaplotypes_old_index_key[pmo_index][ representative_microhaplotypes_index diff --git a/src/pmotools/scripts/pmo_to_tables/extract_allele_table.py b/src/pmotools/scripts/pmo_to_tables/extract_allele_table.py index bfd458b..93f8144 100755 --- a/src/pmotools/scripts/pmo_to_tables/extract_allele_table.py +++ b/src/pmotools/scripts/pmo_to_tables/extract_allele_table.py @@ -91,7 +91,7 @@ def get_parser() -> argparse.ArgumentParser: "--default_base_col_names", type=str, required=False, - default="library_sample_name,target_name,mhap_id", + default="library_sample_name,target_name,seq", help="default base column names, must be length 3", ) return parser diff --git a/tests/data/minimum_Furstenau2025_PMO.json.gz b/tests/data/minimum_Furstenau2025_PMO.json.gz new file mode 100644 index 0000000..b87e2e6 Binary files /dev/null and b/tests/data/minimum_Furstenau2025_PMO.json.gz differ diff --git a/tests/test_pmo_builder/test_metatable_to_pmo.py b/tests/test_pmo_builder/test_metatable_to_pmo.py index 38d59bc..9665dca 100644 --- a/tests/test_pmo_builder/test_metatable_to_pmo.py +++ b/tests/test_pmo_builder/test_metatable_to_pmo.py @@ -219,6 +219,81 @@ def test_add_plate_info_adds_nothing(self): result, [{"specimen_name": "sample1"}, {"specimen_name": "sample2"}] ) + def test_add_plate_info_row_col_skips_blank_cells(self): + df = pd.DataFrame( + { + "specimen_name": ["sample1", "sample2"], + "plate_row": ["A", None], + "plate_col": [1, None], + "plate_name": ["Plate1", None], + } + ) + result = add_plate_info( + "plate_col", + "plate_name", + "plate_row", + None, + self.small_json_example, + df, + "specimen_name", + ) + self.assertEqual( + result[0]["plate_info"], + {"plate_name": "Plate1", "plate_row": "A", "plate_col": 1}, + ) + self.assertNotIn("plate_info", result[1]) + + def test_add_plate_info_position_skips_blank_cells(self): + df = pd.DataFrame( + { + "specimen_name": ["sample1", "sample2", "sample3"], + "plate_position": ["A01", None, " "], + "plate_name": ["Plate1", float("nan"), ""], + } + ) + json_example = [ + {"specimen_name": "sample1"}, + {"specimen_name": "sample2"}, + {"specimen_name": "sample3"}, + ] + result = add_plate_info( + None, + "plate_name", + None, + "plate_position", + json_example, + df, + "specimen_name", + ) + self.assertEqual( + result[0]["plate_info"], + {"plate_name": "Plate1", "plate_row": "A", "plate_col": 1}, + ) + self.assertNotIn("plate_info", result[1]) + self.assertNotIn("plate_info", result[2]) + + def test_add_plate_info_position_fails_with_invalid_value_among_blanks(self): + df = pd.DataFrame( + { + "specimen_name": ["sample1", "sample2"], + "plate_position": [None, "K10"], + } + ) + with self.assertRaises(ValueError) as context: + add_plate_info( + None, + None, + None, + "plate_position", + self.small_json_example, + df, + "specimen_name", + ) + self.assertEqual( + "Values in 'plate_position' must start with a single letter A-H/a-h followed by number 1-12.", + str(context.exception), + ) + def test_add_parasite_density_info_single_value(self): df = pd.DataFrame( { @@ -1028,7 +1103,7 @@ def test_specimen_info_table_to_pmo_with_host_fields(self): "project_name": ["project1", "project2", "project3"], "host_age": [25, None, 30], "host_sex": ["M", "F", ""], - "host_subject_id": ["SUB001", "", "SUB003"], + "host_subject_name": ["SUB001", "", "SUB003"], "gravid": [True, None, False], "gravidity": [2, None, 0], } @@ -1044,7 +1119,7 @@ def test_specimen_info_table_to_pmo_with_host_fields(self): project_name_col="project_name", host_age_col="host_age", host_sex_col="host_sex", - host_subject_id="host_subject_id", + host_subject_name_col="host_subject_name", gravid_col="gravid", gravidity_col="gravidity", ) @@ -1052,25 +1127,25 @@ def test_specimen_info_table_to_pmo_with_host_fields(self): # sample1: all fields have values self.assertEqual(result[0]["host_age"], 25) self.assertEqual(result[0]["host_sex"], "M") - self.assertEqual(result[0]["host_subject_id"], "SUB001") + self.assertEqual(result[0]["host_subject_name"], "SUB001") self.assertEqual(result[0]["gravid"], True) self.assertEqual(result[0]["gravidity"], 2) - # sample2: host_age is None, host_subject_id is empty string, gravid is None, gravidity is None - # Should remove: host_age, host_subject_id, gravid, gravidity + # sample2: host_age is None, host_subject_name is empty string, gravid is None, gravidity is None + # Should remove: host_age, host_subject_name, gravid, gravidity # Should keep: host_sex self.assertNotIn("host_age", result[1]) self.assertEqual(result[1]["host_sex"], "F") - self.assertNotIn("host_subject_id", result[1]) + self.assertNotIn("host_subject_name", result[1]) self.assertNotIn("gravid", result[1]) self.assertNotIn("gravidity", result[1]) # sample3: host_sex is empty string # Should remove: host_sex - # Should keep: host_age, host_subject_id, gravid, gravidity + # Should keep: host_age, host_subject_name, gravid, gravidity self.assertEqual(result[2]["host_age"], 30) self.assertNotIn("host_sex", result[2]) - self.assertEqual(result[2]["host_subject_id"], "SUB003") + self.assertEqual(result[2]["host_subject_name"], "SUB003") self.assertEqual(result[2]["gravid"], False) self.assertEqual(result[2]["gravidity"], 0) @@ -1603,13 +1678,18 @@ def test_library_sample_info_table_to_pmo_with_parasite_density(self): parasite_density_method_col="parasite_density_method", ) - self.assertEqual(result[0]["parasite_density_info"][0]["parasite_density"], 10) self.assertEqual( - result[0]["parasite_density_info"][0]["parasite_density_method"], "qPCR" + result[0]["qpcr_parasite_density_info"][0]["parasite_density"], 10 ) - self.assertEqual(result[1]["parasite_density_info"][0]["parasite_density"], 100) self.assertEqual( - result[1]["parasite_density_info"][0]["parasite_density_method"], + result[0]["qpcr_parasite_density_info"][0]["parasite_density_method"], + "qPCR", + ) + self.assertEqual( + result[1]["qpcr_parasite_density_info"][0]["parasite_density"], 100 + ) + self.assertEqual( + result[1]["qpcr_parasite_density_info"][0]["parasite_density_method"], "microscopy", ) @@ -1638,10 +1718,18 @@ def test_library_sample_info_table_to_pmo_with_parasite_density_multiple(self): parasite_density_method_col=["method1", "method2"], ) - self.assertEqual(result[0]["parasite_density_info"][0]["parasite_density"], 15) - self.assertEqual(result[0]["parasite_density_info"][1]["parasite_density"], 10) - self.assertEqual(result[1]["parasite_density_info"][0]["parasite_density"], 107) - self.assertEqual(result[1]["parasite_density_info"][1]["parasite_density"], 100) + self.assertEqual( + result[0]["qpcr_parasite_density_info"][0]["parasite_density"], 15 + ) + self.assertEqual( + result[0]["qpcr_parasite_density_info"][1]["parasite_density"], 10 + ) + self.assertEqual( + result[1]["qpcr_parasite_density_info"][0]["parasite_density"], 107 + ) + self.assertEqual( + result[1]["qpcr_parasite_density_info"][1]["parasite_density"], 100 + ) def test_library_sample_info_table_to_pmo_with_all_new_fields(self): """Test all new optional fields together""" @@ -1685,7 +1773,9 @@ def test_library_sample_info_table_to_pmo_with_all_new_fields(self): self.assertEqual(result[0]["experiment_accession"], "EXP001") self.assertEqual(result[0]["fastqs_loc"], "/path/to/fastqs1") self.assertEqual(result[0]["run_accession"], "RUN001") - self.assertEqual(result[0]["parasite_density_info"][0]["parasite_density"], 10) + self.assertEqual( + result[0]["qpcr_parasite_density_info"][0]["parasite_density"], 10 + ) self.assertIn("library_prep_plate_info", result[0]) self.assertEqual(result[0]["library_prep_plate_info"]["plate_col"], 1) @@ -1693,7 +1783,9 @@ def test_library_sample_info_table_to_pmo_with_all_new_fields(self): self.assertEqual(result[1]["experiment_accession"], "EXP002") self.assertEqual(result[1]["fastqs_loc"], "/path/to/fastqs2") self.assertEqual(result[1]["run_accession"], "RUN002") - self.assertEqual(result[1]["parasite_density_info"][0]["parasite_density"], 100) + self.assertEqual( + result[1]["qpcr_parasite_density_info"][0]["parasite_density"], 100 + ) self.assertIn("library_prep_plate_info", result[1]) self.assertEqual(result[1]["library_prep_plate_info"]["plate_col"], 2) diff --git a/tests/test_pmo_builder/test_mhap_table_to_pmo.py b/tests/test_pmo_builder/test_mhap_table_to_pmo.py index fad711b..b50903c 100644 --- a/tests/test_pmo_builder/test_mhap_table_to_pmo.py +++ b/tests/test_pmo_builder/test_mhap_table_to_pmo.py @@ -510,52 +510,49 @@ def test_create_representative_microhaplotype_dict_with_ad_cols(self): mhap_table_ad_cols = self.small_mhap_table.copy() mhap_table_ad_cols["adcol1"] = "this" mhap_table_ad_cols["adcol2"] = "that" - mhap_table_ad_cols["cig"] = "cigs" rep_dict_with_ad_cols = copy.deepcopy(self.small_representative_dict) - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][0]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][0]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][0][ - "pseudo_cigar" - ] = "cigs" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][1]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][1]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][1][ - "pseudo_cigar" - ] = "cigs" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][2]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][2]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][0]["microhaplotypes"][2][ - "pseudo_cigar" - ] = "cigs" - rep_dict_with_ad_cols["targets"][1]["microhaplotypes"][0]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][1]["microhaplotypes"][0]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][1]["microhaplotypes"][0][ - "pseudo_cigar" - ] = "cigs" - rep_dict_with_ad_cols["targets"][1]["microhaplotypes"][1]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][1]["microhaplotypes"][1]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][1]["microhaplotypes"][1][ - "pseudo_cigar" - ] = "cigs" - rep_dict_with_ad_cols["targets"][2]["microhaplotypes"][0]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][2]["microhaplotypes"][0]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][2]["microhaplotypes"][0][ - "pseudo_cigar" - ] = "cigs" - rep_dict_with_ad_cols["targets"][2]["microhaplotypes"][1]["adcol1"] = "this" - rep_dict_with_ad_cols["targets"][2]["microhaplotypes"][1]["adcol2"] = "that" - rep_dict_with_ad_cols["targets"][2]["microhaplotypes"][1][ - "pseudo_cigar" - ] = "cigs" + for target in rep_dict_with_ad_cols["targets"]: + for mhap in target["microhaplotypes"]: + mhap["adcol1"] = "this" + mhap["adcol2"] = "that" actual = create_representative_microhaplotype_dict( mhap_table_ad_cols, additional_representative_mhap_cols=["adcol1", "adcol2"], - pseudocigar_col="cig", ) self.assertEqual(actual, rep_dict_with_ad_cols) + def test_create_representative_microhaplotype_dict_with_pseudocigar(self): + tbl = self.small_mhap_table.copy() + tbl["chrom"] = "chr1" + tbl["start"] = 100 + tbl["end"] = 150 + tbl["cig"] = "8M" + actual = create_representative_microhaplotype_dict( + tbl, + chrom_col="chrom", + start_col="start", + end_col="end", + pseudocigar_col="cig", + pseudocigar_start_col="start", + pseudocigar_end_col="end", + ) + first = actual["targets"][0]["microhaplotypes"][0] + # pseudocigar is a Pseudocigar object with a GenomicLocation ref_loc; + # the chromosome is reused from chrom_col and genome_id defaults to 0 + self.assertEqual(first["pseudocigar"]["pseudocigar_seq"], "8M") + self.assertEqual( + first["pseudocigar"]["ref_loc"], + {"genome_id": 0, "chrom": "chr1", "start": 100, "end": 150}, + ) + + def test_pseudocigar_without_ref_loc_columns_raises(self): + tbl = self.small_mhap_table.copy() + tbl["cig"] = "8M" + with self.assertRaises(ValueError): + create_representative_microhaplotype_dict(tbl, pseudocigar_col="cig") + @patch( "pmotools.pmo_builder.mhap_table_to_pmo.create_representative_microhaplotype_dict" ) diff --git a/tests/test_pmo_builder/test_panel_information_to_pmo.py b/tests/test_pmo_builder/test_panel_information_to_pmo.py index 29158a3..bf74cf3 100644 --- a/tests/test_pmo_builder/test_panel_information_to_pmo.py +++ b/tests/test_pmo_builder/test_panel_information_to_pmo.py @@ -416,8 +416,8 @@ def test_build_target_info_dict_full_info( "location": { "genome_id": 0, "chrom": "chrom1", - "end": 1, - "start": 2, + "start": 1, + "end": 2, }, }, "reverse_primer": { @@ -443,8 +443,8 @@ def test_build_target_info_dict_full_info( "location": { "genome_id": 0, "chrom": "chrom1", - "end": 1, - "start": 2, + "start": 1, + "end": 2, }, }, "reverse_primer": { @@ -470,8 +470,8 @@ def test_build_target_info_dict_full_info( "location": { "genome_id": 0, "chrom": "chrom1", - "end": 1, - "start": 2, + "start": 1, + "end": 2, }, }, "reverse_primer": { @@ -526,8 +526,8 @@ def test_build_target_info_dict_missing_info( "location": { "genome_id": 0, "chrom": "chrom1", - "end": 1, - "start": 2, + "start": 1, + "end": 2, }, }, "reverse_primer": {"seq": "GTT"}, diff --git a/tests/test_pmo_builder/test_pmo_updater.py b/tests/test_pmo_builder/test_pmo_updater.py index 930fa28..447474c 100755 --- a/tests/test_pmo_builder/test_pmo_updater.py +++ b/tests/test_pmo_builder/test_pmo_updater.py @@ -152,11 +152,11 @@ def test_update_specimen_meta_with_traveler_info_with_optional(self): "travel_country": "Kenya", "travel_start_date": "2024-01", "travel_end_date": "2024-01-20", - "bed_net": 0.5, - "admin1": "Nairobi", - "admin2": "SubCounty1", - "admin3": "Ward1", - "latlon": "-1.2921,36.8219", + "bed_net_usage": 0.5, + "geo_admin1": "Nairobi", + "geo_admin2": "SubCounty1", + "geo_admin3": "Ward1", + "lat_lon": "-1.2921,36.8219", } ], }, @@ -167,11 +167,11 @@ def test_update_specimen_meta_with_traveler_info_with_optional(self): "travel_country": "Tanzania", "travel_start_date": "2024-02", "travel_end_date": "2024-02-15", - "bed_net": 0.0, - "admin1": "Dar es Salaam", - "admin2": "SubCounty2", - "admin3": "Ward2", - "latlon": "-6.7924,39.2083", + "bed_net_usage": 0.0, + "geo_admin1": "Dar es Salaam", + "geo_admin2": "SubCounty2", + "geo_admin3": "Ward2", + "lat_lon": "-6.7924,39.2083", } ], }, @@ -233,11 +233,11 @@ def test_update_specimen_meta_with_traveler_info_with_optional_replace_old(self) "travel_country": "Kenya", "travel_start_date": "2024-01", "travel_end_date": "2024-01-20", - "bed_net": 0.5, - "admin1": "Nairobi", - "admin2": "SubCounty1", - "admin3": "Ward1", - "latlon": "-1.2921,36.8219", + "bed_net_usage": 0.5, + "geo_admin1": "Nairobi", + "geo_admin2": "SubCounty1", + "geo_admin3": "Ward1", + "lat_lon": "-1.2921,36.8219", } ], }, @@ -248,11 +248,11 @@ def test_update_specimen_meta_with_traveler_info_with_optional_replace_old(self) "travel_country": "Tanzania", "travel_start_date": "2024-02", "travel_end_date": "2024-02-15", - "bed_net": 0.0, - "admin1": "Dar es Salaam", - "admin2": "SubCounty2", - "admin3": "Ward2", - "latlon": "-6.7924,39.2083", + "bed_net_usage": 0.0, + "geo_admin1": "Dar es Salaam", + "geo_admin2": "SubCounty2", + "geo_admin3": "Ward2", + "lat_lon": "-6.7924,39.2083", } ], }, @@ -260,6 +260,273 @@ def test_update_specimen_meta_with_traveler_info_with_optional_replace_old(self) } self.assertEqual(test_out_pmo, test_pmo) + def test_update_specimen_meta_with_traveler_info_skips_blank_optional(self): + test_pmo = {"specimen_info": [{"specimen_name": "spec1"}]} + traveler_info = pd.DataFrame( + { + "specimen_name": ["spec1"], + "travel_country": ["Kenya"], + "travel_start_date": ["2024-01"], + "travel_end_date": ["2024-02"], + "bed_net": [None], + "admin1": [""], + } + ) + PMOUpdater.update_specimen_meta_with_traveler_info( + test_pmo, + traveler_info, + bed_net_usage_col="bed_net", + geo_admin1_col="admin1", + ) + self.assertEqual( + test_pmo["specimen_info"][0]["travel_out_six_month"], + [ + { + "travel_country": "Kenya", + "travel_start_date": "2024-01", + "travel_end_date": "2024-02", + } + ], + ) + + def test_update_target_info_with_markers_of_interest(self): + test_pmo = { + "target_info": [{"target_name": "t1"}, {"target_name": "t2"}], + } + markers_info = pd.DataFrame( + { + "target_name": ["t1", "t1"], + "chrom": ["Pf3D7_04_v3", "Pf3D7_04_v3"], + "start": [748238, 748409], + "end": [748239, 748410], + "ref_seq": ["A", ""], + "associations": ["SP resistance, dhfr", None], + } + ) + PMOUpdater.update_target_info_with_markers_of_interest( + test_pmo, + markers_info, + ref_seq_col="ref_seq", + associations_col="associations", + ) + self.assertEqual( + test_pmo["target_info"][0]["markers_of_interest"], + [ + { + "marker_location": { + "genome_id": 0, + "chrom": "Pf3D7_04_v3", + "start": 748238, + "end": 748239, + "ref_seq": "A", + }, + "associations": ["SP resistance", "dhfr"], + }, + { + "marker_location": { + "genome_id": 0, + "chrom": "Pf3D7_04_v3", + "start": 748409, + "end": 748410, + } + }, + ], + ) + self.assertNotIn("markers_of_interest", test_pmo["target_info"][1]) + + def test_update_target_info_with_markers_of_interest_replace(self): + old_marker = { + "marker_location": {"genome_id": 0, "chrom": "c", "start": 1, "end": 2} + } + markers_info = pd.DataFrame( + {"target_name": ["t1"], "chrom": ["c"], "start": [5], "end": [6]} + ) + for replace, expected_count in ((False, 2), (True, 1)): + test_pmo = { + "target_info": [ + {"target_name": "t1", "markers_of_interest": [dict(old_marker)]} + ] + } + PMOUpdater.update_target_info_with_markers_of_interest( + test_pmo, markers_info, replace_current_markers=replace + ) + self.assertEqual( + len(test_pmo["target_info"][0]["markers_of_interest"]), + expected_count, + ) + + def test_update_target_info_with_markers_of_interest_raises(self): + test_pmo = {"target_info": [{"target_name": "t1"}]} + unknown_target = pd.DataFrame( + {"target_name": ["t9"], "chrom": ["c"], "start": [1], "end": [2]} + ) + with self.assertRaises(ValueError): + PMOUpdater.update_target_info_with_markers_of_interest( + test_pmo, unknown_target + ) + blank_start = pd.DataFrame( + {"target_name": ["t1"], "chrom": ["c"], "start": [None], "end": [2]} + ) + with self.assertRaises(ValueError): + PMOUpdater.update_target_info_with_markers_of_interest( + test_pmo, blank_start + ) + with self.assertRaises(ValueError): + PMOUpdater.update_target_info_with_markers_of_interest( + test_pmo, unknown_target, chrom_col="not_a_column" + ) + + def _rep_mhap_pmo(self): + return { + "target_info": [{"target_name": "t1"}, {"target_name": "t2"}], + "representative_microhaplotypes": { + "targets": [ + { + "target_id": 1, + "microhaplotypes": [{"seq": "ACGT"}, {"seq": "ACTT"}], + } + ] + }, + } + + def test_update_representative_microhaplotypes_with_seq_variants(self): + test_pmo = self._rep_mhap_pmo() + seq_variants_info = pd.DataFrame( + { + "target_name": ["t2", "t2"], + "seq": ["ACTT", "ACTT"], + "chrom": ["c", "c"], + "start": [2, 3], + "end": [3, 4], + "ref_seq": ["G", "T"], + "alt_seq": ["T", None], + } + ) + PMOUpdater.update_representative_microhaplotypes_with_seq_variants( + test_pmo, + seq_variants_info, + ref_seq_col="ref_seq", + alt_seq_col="alt_seq", + ) + mhaps = test_pmo["representative_microhaplotypes"]["targets"][0][ + "microhaplotypes" + ] + self.assertNotIn("associated_seq_variants", mhaps[0]) + self.assertEqual( + mhaps[1]["associated_seq_variants"], + [ + { + "genome_id": 0, + "chrom": "c", + "start": 2, + "end": 3, + "ref_seq": "G", + "alt_seq": "T", + }, + {"genome_id": 0, "chrom": "c", "start": 3, "end": 4, "ref_seq": "T"}, + ], + ) + + def test_update_representative_microhaplotypes_with_seq_variants_raises(self): + for target_name, seq in (("t2", "GGGG"), ("t1", "ACGT")): + seq_variants_info = pd.DataFrame( + { + "target_name": [target_name], + "seq": [seq], + "chrom": ["c"], + "start": [1], + "end": [2], + } + ) + with self.assertRaises(ValueError): + PMOUpdater.update_representative_microhaplotypes_with_seq_variants( + self._rep_mhap_pmo(), seq_variants_info + ) + + def test_update_representative_microhaplotypes_with_protein_variants(self): + test_pmo = self._rep_mhap_pmo() + protein_variants_info = pd.DataFrame( + { + "target_name": ["t2", "t2"], + "seq": ["ACGT", "ACGT"], + "transcript": ["PF3D7_0417200.1", "PF3D7_0417200.1"], + "protein_start": [50, 107], + "protein_end": [51, 108], + "protein_ref": ["C", "S"], + "protein_alt": ["R", "N"], + "gene": ["dhfr", ""], + "codon_chrom": ["Pf3D7_04_v3", None], + "codon_start": [748238, None], + "codon_end": [748241, None], + } + ) + PMOUpdater.update_representative_microhaplotypes_with_protein_variants( + test_pmo, + protein_variants_info, + protein_ref_seq_col="protein_ref", + protein_alt_seq_col="protein_alt", + gene_name_col="gene", + codon_chrom_col="codon_chrom", + codon_start_col="codon_start", + codon_end_col="codon_end", + ) + mhaps = test_pmo["representative_microhaplotypes"]["targets"][0][ + "microhaplotypes" + ] + self.assertEqual( + mhaps[0]["associated_protein_variants"], + [ + { + "protein_location": { + "genome_id": 0, + "chrom": "PF3D7_0417200.1", + "start": 50, + "end": 51, + "ref_seq": "C", + "alt_seq": "R", + }, + "gene_name": "dhfr", + "codon_genomic_location": { + "genome_id": 0, + "chrom": "Pf3D7_04_v3", + "start": 748238, + "end": 748241, + }, + }, + { + "protein_location": { + "genome_id": 0, + "chrom": "PF3D7_0417200.1", + "start": 107, + "end": 108, + "ref_seq": "S", + "alt_seq": "N", + }, + }, + ], + ) + self.assertNotIn("associated_protein_variants", mhaps[1]) + + def test_update_representative_microhaplotypes_with_protein_variants_partial_codon_cols( + self, + ): + protein_variants_info = pd.DataFrame( + { + "target_name": ["t2"], + "seq": ["ACGT"], + "transcript": ["tx"], + "protein_start": [1], + "protein_end": [2], + "codon_chrom": ["c"], + } + ) + with self.assertRaises(ValueError): + PMOUpdater.update_representative_microhaplotypes_with_protein_variants( + self._rep_mhap_pmo(), + protein_variants_info, + codon_chrom_col="codon_chrom", + ) + # PMOUpdater.merge_dicts_by_key def test_merge_dicts_by_key_correct_fields_added(self): result = PMOUpdater.merge_dicts_by_key( diff --git a/tests/test_pmo_builder/test_schema_validation_integration.py b/tests/test_pmo_builder/test_schema_validation_integration.py index 766bca5..5f88b9b 100644 --- a/tests/test_pmo_builder/test_schema_validation_integration.py +++ b/tests/test_pmo_builder/test_schema_validation_integration.py @@ -191,6 +191,8 @@ def test_full_1_0_0_toy_pmo_validates_against_schema(): umis_col="umis", microhaplotype_name_col="microhap_name", pseudocigar_col="pseudocigar", + pseudocigar_start_col="start", + pseudocigar_end_col="end", quality_col="quality", masking_seq_start_col="mask_start", masking_seq_segment_size_col="mask_segment", @@ -467,6 +469,9 @@ def test_slimer_1_1_0_toy_pmo_validates_against_schema(): "umis": [10], "microhap_name": ["mh1"], "pseudocigar": ["8M"], + "pseudocigar_chrom": ["chr1"], + "pseudocigar_start": [120], + "pseudocigar_end": [127], "quality": ["ABCD"], "mask_start": ["1"], "mask_segment": ["2"], @@ -480,6 +485,9 @@ def test_slimer_1_1_0_toy_pmo_validates_against_schema(): umis_col="umis", microhaplotype_name_col="microhap_name", pseudocigar_col="pseudocigar", + pseudocigar_chrom_col="pseudocigar_chrom", + pseudocigar_start_col="pseudocigar_start", + pseudocigar_end_col="pseudocigar_end", quality_col="quality", masking_seq_start_col="mask_start", masking_seq_segment_size_col="mask_segment", diff --git a/tests/test_pmo_engine/test_pmo_exporter.py b/tests/test_pmo_engine/test_pmo_exporter.py index 1984bfa..44e2389 100755 --- a/tests/test_pmo_engine/test_pmo_exporter.py +++ b/tests/test_pmo_engine/test_pmo_exporter.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import copy import gzip import hashlib import os @@ -38,6 +39,15 @@ def setUp(self): "rt", ) as f: self.minimum_fields_v1_1_0_pmo_data = json.load(f) + # a real, schema-valid v1.1.0 PMO with only the required top-level sections + with gzip.open( + os.path.join( + os.path.dirname(self.working_dir), + "data/minimum_Furstenau2025_PMO.json.gz", + ), + "rt", + ) as f: + self.minimum_v1_1_0_pmo_data = json.load(f) def tearDown(self): self.test_dir.cleanup() @@ -105,6 +115,54 @@ def test_extract_panels_insert_bed_loc(self): PMOExporter.write_bed_locs(all_target_inserts, output_fnp) self.assertEqual("52b1f79a3a89f8265573fa54b5a7ce57", md5sum_of_fnp(output_fnp)) + def test_extract_panels_insert_bed_loc_covers_all_reactions(self): + # regression: previously the function returned early inside the reaction + # loop, so only the first reaction of the first panel was returned. + import copy + + pmo = copy.deepcopy(self.combined_pmo_data) + panel = pmo["panel_info"][0] + targets = panel["reactions"][0]["panel_targets"] + half = len(targets) // 2 + panel["reactions"] = [ + {"reaction_name": "pool1", "panel_targets": targets[:half]}, + {"reaction_name": "pool2", "panel_targets": targets[half:]}, + ] + bed_locs = PMOExporter.extract_panels_insert_bed_loc(pmo, sort_output=False) + # every target across BOTH reactions is present + self.assertEqual(len(bed_locs), len(targets)) + self.assertEqual( + {b.name for b in bed_locs}, + {pmo["target_info"][t]["target_name"] for t in targets}, + ) + + def test_write_bed_locs_header_on_own_line(self): + # regression: header was written without a trailing newline, gluing the + # first data row onto it. + bed_locs = PMOExporter.extract_targets_insert_bed_loc( + self.combined_pmo_data, sort_output=True + ) + out_fnp = os.path.join(self.test_dir.name, "with_header.bed") + PMOExporter.write_bed_locs(bed_locs, out_fnp, add_header=True) + with open(out_fnp) as f: + lines = f.read().splitlines() + self.assertEqual( + lines[0], + "\t".join( + [ + "#chrom", + "start", + "end", + "name", + "score", + "strand", + "ref_seq", + "extra_info", + ] + ), + ) + self.assertEqual(len(lines), len(bed_locs) + 1) + def test_extract_alleles_per_sample_table(self): allele_data = PMOExporter.extract_alleles_per_sample_table( self.combined_pmo_data, @@ -245,6 +303,28 @@ def test_export_library_sample_meta_table(self): md5sum_of_fnp(os.path.join(self.test_dir.name, "library_sample_table.csv")), ) + def test_export_meta_tables_without_optional_cross_ref_sections(self): + # project_info and sequencing_info are optional as of v1.1.0; the specimen and + # library_sample meta-table exporters must not raise a raw KeyError when a + # referencing id is present but the referenced optional section is absent + self.assertNotIn("project_info", self.minimum_v1_1_0_pmo_data) + self.assertNotIn("sequencing_info", self.minimum_v1_1_0_pmo_data) + # a clean minimal PMO (no dangling ids) should export fine + PMOExporter.export_specimen_meta_table(self.minimum_v1_1_0_pmo_data) + PMOExporter.export_library_sample_meta_table(self.minimum_v1_1_0_pmo_data) + + # inject the referencing ids without the optional sections (referential dangling) + # the raw id is kept rather than resolving an unavailable name + dangling = copy.deepcopy(self.minimum_v1_1_0_pmo_data) + dangling["specimen_info"][0]["project_id"] = 0 + dangling["library_sample_info"][0]["sequencing_info_id"] = 0 + spec_table = PMOExporter.export_specimen_meta_table(dangling) + self.assertIn("project_id", spec_table.columns) + self.assertNotIn("project_name", spec_table.columns) + library_table = PMOExporter.export_library_sample_meta_table(dangling) + self.assertIn("sequencing_info_id", library_table.columns) + self.assertNotIn("sequencing_info_name", library_table.columns) + def test_export_sequencing_info_meta_table(self): sequencing_info_table = PMOExporter.export_sequencing_info_meta_table( self.small_example_pmo_data diff --git a/tests/test_pmo_engine/test_pmo_processor.py b/tests/test_pmo_engine/test_pmo_processor.py index 2c4a41b..729f64d 100755 --- a/tests/test_pmo_engine/test_pmo_processor.py +++ b/tests/test_pmo_engine/test_pmo_processor.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +import copy +import gzip import os import tempfile import unittest @@ -36,6 +38,16 @@ def setUp(self): ) ) as f: self.minimum_pmo_data = json.load(f) + # a real, schema-valid v1.1.0 PMO that contains only the required top-level + # sections (no sequencing_info/project_info/targeted_genomes/bioinformatics_*/ + # read_counts_by_stage) to guard against unguarded optional-section access + with gzip.open( + os.path.join( + os.path.dirname(self.working_dir), + "data/minimum_Furstenau2025_PMO.json.gz", + ) + ) as f: + self.minimum_v1_1_0_pmo_data = json.load(f) def tearDown(self): self.test_dir.cleanup() @@ -317,10 +329,52 @@ def test_extract_from_pmo_with_read_filter(self): ) with open(output_fnp, "w") as f: json.dump(pmo_data_filtered, f) - self.assertEqual("879b1a0c62a77a8fcc910a21d8ec9de6", md5sum_of_fnp(output_fnp)) + self.assertEqual("59e24e7efd4cd7822acde70828ad6119", md5sum_of_fnp(output_fnp)) checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_filtered) + def test_extract_from_pmo_with_read_filter_missing_optional_sections(self): + # the v1.1.0 schema relaxed the required fields, so a PMO may omit these + # optional sections; the extracted PMO must not copy them in as null/empty keys + optional_sections = [ + "sequencing_info", + "project_info", + "targeted_genomes", + "bioinformatics_methods_info", + "bioinformatics_run_info", + "read_counts_by_stage", + ] + stripped_pmo_data = copy.deepcopy(self.combined_pmo_data) + for section in optional_sections: + stripped_pmo_data.pop(section, None) + pmo_data_filtered = PMOProcessor.extract_from_pmo_with_read_filter( + stripped_pmo_data, 1000 + ) + for section in optional_sections: + self.assertNotIn(section, pmo_data_filtered) + # the relaxed required fields are a v1.1.0 change, so validate against v1.1.0 + checker = PMOChecker( + load_schema("portable_microhaplotype_object_v1.1.0.schema.json") + ) + checker.validate_pmo_json(pmo_data_filtered) + + def test_bioinformatics_run_name_lookups_without_run_info(self): + # bioinformatics_run_info is optional as of v1.1.0; the run-name lookups + # must not raise a raw KeyError on a valid PMO that omits it + self.assertNotIn("bioinformatics_run_info", self.minimum_v1_1_0_pmo_data) + self.assertEqual( + {}, + PMOProcessor.get_index_key_of_bioinformatics_run_names( + self.minimum_v1_1_0_pmo_data + ), + ) + self.assertEqual( + [], + PMOProcessor.get_sorted_bioinformatics_run_names( + self.minimum_v1_1_0_pmo_data + ), + ) + def test_filter_pmo_by_target_ids(self): pmo_data_select_targets = PMOProcessor.filter_pmo_by_target_ids( self.combined_pmo_data, {1, 10, 11, 55} @@ -358,7 +412,7 @@ def test_filter_pmo_by_library_sample_ids(self): ) with open(output_fnp, "w") as f: json.dump(pmo_data_select_targets, f) - self.assertEqual("4a2c7b9d84322cc43adf2b9f0c518244", md5sum_of_fnp(output_fnp)) + self.assertEqual("36d5cb8a79962707374fae86ba1a6f90", md5sum_of_fnp(output_fnp)) # check pmo extracted against PMO schema checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_select_targets) @@ -374,7 +428,7 @@ def test_filter_pmo_by_library_sample_names(self): ) with open(output_fnp, "w") as f: json.dump(pmo_data_select_library_sample_names, f) - self.assertEqual("ffdaeb6b57ed9a5c51e56cfaa4621796", md5sum_of_fnp(output_fnp)) + self.assertEqual("6bd4e6367a62f1eb3c73f481a5ae9013", md5sum_of_fnp(output_fnp)) # check pmo extracted against PMO schema checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_select_library_sample_names) @@ -388,7 +442,7 @@ def test_filter_pmo_by_specimen_ids(self): ) with open(output_fnp, "w") as f: json.dump(pmo_data_select_targets, f) - self.assertEqual("f3dd2e0aee9afa82336ff48475a3c69c", md5sum_of_fnp(output_fnp)) + self.assertEqual("664de0f14180de852f3f5d8942454004", md5sum_of_fnp(output_fnp)) # check pmo extracted against PMO schema checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_select_targets) @@ -402,7 +456,7 @@ def test_filter_pmo_by_specimen_names(self): ) with open(output_fnp, "w") as f: json.dump(pmo_data_select_targets, f) - self.assertEqual("f3dd2e0aee9afa82336ff48475a3c69c", md5sum_of_fnp(output_fnp)) + self.assertEqual("664de0f14180de852f3f5d8942454004", md5sum_of_fnp(output_fnp)) # check pmo extracted against PMO schema checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_select_targets) @@ -420,7 +474,7 @@ def test_extract_from_pmo_samples_with_meta_groupings(self): with open(output_fnp, "w") as f: json.dump(pmo_data_select_meta, f) - self.assertEqual("f0f67e399885824a7aa318ead8b3d09c", md5sum_of_fnp(output_fnp)) + self.assertEqual("9deb0bccc0017593697190d45ae6c0a4", md5sum_of_fnp(output_fnp)) # check pmo extracted against PMO schema checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_select_meta) diff --git a/tests/test_pmo_engine/test_pmo_reader.py b/tests/test_pmo_engine/test_pmo_reader.py index b612ae2..0075a48 100755 --- a/tests/test_pmo_engine/test_pmo_reader.py +++ b/tests/test_pmo_engine/test_pmo_reader.py @@ -165,6 +165,73 @@ def test_combine_multiple_pmos_fail_dup_library_sample_names(self): ) self.assertRaises(Exception, PMOReader.combine_multiple_pmos, pmo_data_list_2) + def test_combine_multiple_pmos_remaps_new_rep_target_id(self): + # a representative-microhaplotype target newly added from a + # non-first PMO must have its target_id remapped to the combined + # target_info (previously it kept its local target_id), this test + # ensures this is tested as it was previously untested (2026-06-29) + import pandas as pd + from pmotools.pmo_builder.mhap_table_to_pmo import mhap_table_to_pmo + from pmotools.pmo_builder.panel_information_to_pmo import ( + panel_info_table_to_pmo, + ) + from pmotools.pmo_builder.merge_to_pmo import merge_to_pmo + + def build(libs, targets, seqs, panel_name, panel_targets): + mhap_info = mhap_table_to_pmo( + pd.DataFrame( + { + "library_sample_name": libs, + "target_name": targets, + "seq": seqs, + "reads": [10] * len(libs), + } + ) + ) + panel_info = panel_info_table_to_pmo( + pd.DataFrame( + { + "target_name": panel_targets, + "fwd_primer": [ + "A" * (i + 4) for i in range(len(panel_targets)) + ], + "rev_primer": [ + "C" * (i + 4) for i in range(len(panel_targets)) + ], + } + ), + panel_name, + ) + return merge_to_pmo(mhap_info=mhap_info, panel_target_info=panel_info) + + pmo_a = build( + ["S1", "S2"], ["t1", "t2"], ["AAA", "GGG"], "panelA", ["t1", "t2"] + ) + # pmo_b introduces a brand-new target t3 + pmo_b = build( + ["S3", "S4"], ["t1", "t3"], ["AAA", "CCC"], "panelB", ["t1", "t3"] + ) + + combined = PMOReader.combine_multiple_pmos([pmo_a, pmo_b]) + self.assertEqual( + [t["target_name"] for t in combined["target_info"]], + ["t1", "t2", "t3"], + ) + # every rep target's target_id resolves to the right combined target + for rep in combined["representative_microhaplotypes"]["targets"]: + self.assertLess(rep["target_id"], len(combined["target_info"])) + t3_reps = [ + rep + for rep in combined["representative_microhaplotypes"]["targets"] + if combined["target_info"][rep["target_id"]]["target_name"] == "t3" + ] + self.assertEqual(len(t3_reps), 1) + # validate against schema (minimal builder PMO targets v1.1.0) + checker = PMOChecker( + load_schema("portable_microhaplotype_object_v1.1.0.schema.json") + ) + checker.validate_pmo_json(combined) + def test_combine_multiple_pmos_fail_for_combine_only_one_file(self): # will fail for only having 1 PMO pmo_data_list_2 = PMOReader.read_in_pmos( diff --git a/uv.lock b/uv.lock index 033c746..b41df83 100644 --- a/uv.lock +++ b/uv.lock @@ -285,7 +285,7 @@ wheels = [ [[package]] name = "pmotools" -version = "1.0.0" +version = "1.2.0" source = { editable = "." } dependencies = [ { name = "biopython" },