diff --git a/apps/compairr/ls6/Makefile b/apps/compairr/ls6/Makefile index 300abc1..b50cdf0 100644 --- a/apps/compairr/ls6/Makefile +++ b/apps/compairr/ls6/Makefile @@ -1,7 +1,7 @@ # helper commands for tapis v3 apps APP_NAME=compairr -APP_VERSION=0.1 +APP_VERSION=0.2 APP_SYSTEM=ls6 APP_ID=${APP_NAME}-${APP_SYSTEM} @@ -30,13 +30,14 @@ help: @echo "" @echo "Testing:" @echo "make test-job -- Submit test job" + @echo "make test-all -- Submit all test jobs" @echo "" create: - tapis_apps_create ${APP_JSON} + vdjserver-tools apps create ${APP_JSON} update: - tapis_apps_update ${APP_ID} ${APP_VERSION} ${APP_JSON} + vdjserver-tools apps update ${APP_ID} ${APP_VERSION} ${APP_JSON} clean: rm -rf ${BUILD_DIR} ${APP_ARCHIVE} @@ -50,15 +51,19 @@ common-scripts: build: clean common-scripts cp ${APP_JSON} ${BUILD_DIR} cp tapisjob_app.sh ${BUILD_DIR} + cp concatenate_airr_tsv.py ${BUILD_DIR} cp ${APP_NAME}_common.sh ${BUILD_DIR} tar zcf ${APP_ARCHIVE} -C ${BUILD_DIR} . deploy: build - tapis_files_upload /akc/apps/${APP_ARCHIVE} ${APP_ARCHIVE} + vdjserver-tools files upload ${APP_ARCHIVE} /apps/${APP_ARCHIVE} test-job: - tapis_jobs_submit test/test-cli.json + vdjserver-tools jobs submit test/test-cli.json -test-cache: - tapis_jobs_submit test/test-cache.json +test-all: + vdjserver-tools jobs submit test/test-cli.json + vdjserver-tools jobs submit test/test-cluster.json + vdjserver-tools jobs submit test/test-matrix.json + vdjserver-tools jobs submit test/test-overlap.json diff --git a/apps/compairr/ls6/compairr-ls6.json b/apps/compairr/ls6/compairr-ls6.json index 4e5903d..4bb0cc5 100644 --- a/apps/compairr/ls6/compairr-ls6.json +++ b/apps/compairr/ls6/compairr-ls6.json @@ -1,16 +1,19 @@ { "id":"compairr-ls6", - "version":"0.1", + "version":"0.2", "description":"CompAIRR analysis for AKC", "runtime":"ZIP", "runtimeOptions": ["NONE"], - "containerImage":"tapis://data-storage.vdjserver.org/akc/apps/compairr-ls6-0.1.tgz", + "containerImage":"tapis://data-storage.vdjserver.org/apps/compairr-ls6-0.2.tgz", "jobType":"BATCH", "jobAttributes": { "description": "default job description", + "execSystemId": "test-ls6.tacc.utexas.edu", "fileInputs":[ - { "name":"compairr_image", "inputMode": "FIXED", "envKey":"compairr_image", "sourceUrl":"tapis://data-storage.vdjserver.org/singularity/ir_compairr-1.13.0.sif", "targetPath":"ir_compairr-1.13.0.sif" }, - { "name":"airr_tsv_file", "inputMode": "REQUIRED", "envKey":"airr_tsv_file", "targetPath":"." } + { "name":"compairr_image", "inputMode": "FIXED", "envKey":"compairr_image", "sourceUrl":"tapis://data-storage.vdjserver.org/singularity/ir_compairr-1.13.0.sif", "targetPath":"ir_compairr-1.13.0.sif" } + ], + "fileInputArrays":[ + { "name":"airr_tsv_files", "inputMode":"REQUIRED", "targetDir":"."} ], "parameterSet": { "logConfig": { @@ -22,9 +25,9 @@ "appArgs": [ ], "envVariables": [ - {"key": "analysis_type", "inputMode": "REQUIRED", "value": "product" }, - {"key": "distance", "inputMode": "REQUIRED", "value": "1" }, - {"key": "file_type", "inputMode": "REQUIRED", "value": "rearrangement" } + {"key": "airr_tsv_files", "inputMode": "REQUIRED"}, + {"key": "analysis_type", "inputMode": "REQUIRED"}, + {"key": "distance", "inputMode": "INCLUDE_BY_DEFAULT"} ] } } diff --git a/apps/compairr/ls6/compairr_common.sh b/apps/compairr/ls6/compairr_common.sh index e2db3c4..5985604 100644 --- a/apps/compairr/ls6/compairr_common.sh +++ b/apps/compairr/ls6/compairr_common.sh @@ -34,49 +34,58 @@ function print_versions() { function print_parameters() { echo "Input files:" echo "compairr_image=${compairr_image}" - echo "airr_tsv_file=${airr_tsv_file}" + echo "airr_tsv_files=${airr_tsv_files}" echo "" echo "Application parameters:" echo "analysis_type=${analysis_type}" echo "distance=${distance}" - echo "file_type=${file_type}" } function run_compairr_workflow() { initProvenance + #concatenate files + concatenated_file="concat.tsv" + ${PYTHON} concatenate_airr_tsv.py -i $airr_tsv_files -o $concatenated_file + #deduplicate files + deduplicated_file="dedup_${concatenated_file}" + echo "Command: apptainer exec -e ${compairr_image} compairr --deduplicate --out ${deduplicated_file}" + apptainer exec -e "${compairr_image}" compairr --deduplicate --out "${deduplicated_file}" "${concatenated_file}" - # expand rearrangement file if its compressed - expandfile $airr_tsv_file - #noArchive $file - - # Assuming airr.tsv extension - fileBasename="${file%.*}" # file.airr.tsv -> file.airr - fileBasename="${fileBasename%.*}" # file.airr -> file - - # Run compairr in matrix or cluster mode, using the "analysis_type" to determine - # which method to use. $distance is used only in cluster mode. - if [[ "$file_type" == "rearrangement" ]] ; then - if [[ "$analysis_type" == "cluster" ]] ; then - if [[ ! "x$distance" == "x" ]]; then - re='^[0-9]+$' - if [[ $distance =~ $re ]]; then - echo "Runnig: apptainer exec -e ${compairr_image} compairr -f -e -u --cluster ${file} -d ${distance} --out $fileBasename.cluster.tsv" - apptainer exec -e ${compairr_image} compairr -f -e -u --cluster ${file} -d ${distance} --out $fileBasename.cluster.tsv - else - echo "ERROR: Distance metric ($distance) integer and greater than 0 required" - return - fi - else - echo "ERROR: Distance metric not provided" - return - fi - elif [[ "$analysis_type" == "product" || "$analysis_type" == "MH" || "$analysis_type" == "Morisita-Horn" ]] ; then - echo "Runnig: apptainer exec -e ${compairr_image} compairr -f -e -u -s $analysis_type --matrix ${file} --out $fileBasename.matrix.tsv" - apptainer exec -e ${compairr_image} compairr -f -e -u -s $analysis_type --matrix ${file} --out $fileBasename.matrix.tsv - else - echo "ERROR: Invalid analysis type $analysis_type provided" - return - fi - fi + # Assuming .tsv extension + file_basename="${deduplicated_file%.*}" # file.tsv -> file + n_threads=20 + + cluster_file="${file_basename}_d_${distance}_clust.tsv" + + default_matrix_file="${file_basename}_d_${distance}_prodmat.txt" + pairs_file="${file_basename}_d_${distance}_pairs.tsv" + + mh_matrix_file="${file_basename}_MHmat.txt" + jaccard_matrix_file="${file_basename}_Jacmat.txt" + + if [[ "$analysis_type" == "cluster" ]] ; then + echo "Running Cluster Analysis for distance $distance" + + echo "Command: apptainer exec -e ${compairr_image} compairr --cluster ${deduplicated_file} -d ${distance} --out $cluster_file" + apptainer exec -e "${compairr_image}" compairr --cluster -d "${distance}" --threads "${n_threads}" --out "$cluster_file" "${deduplicated_file}" + + elif [[ "$analysis_type" == "overlap" ]]; then + echo "Calculating overlap analysis for distance $distance." + echo "Command: apptainer exec -e ${compairr_image} compairr --matrix -d ${distance} --pairs ${pairs_file} ${deduplicated_file}" + apptainer exec -e "${compairr_image}" compairr --matrix -d "${distance}" --threads "${n_threads}" --pairs "${pairs_file}" "${deduplicated_file}" + + elif [[ "$analysis_type" == "matrix" ]]; then + echo "Running matrix analysis for MH and Jaccard Score." + + echo "Command: apptainer -e ${compairr_image} compairr --matrix --out ${mh_matrix_file} --score MH ${deduplicated_file}" + apptainer exec -e "${compairr_image}" compairr --matrix --threads "${n_threads}" --out "${mh_matrix_file}" --score MH "${deduplicated_file}" + + echo "Command: apptainer exec -e ${compairr_image} compairr --matrix --out ${jaccard_matrix_file} --score Jaccard ${deduplicated_file} " + apptainer exec -e "${compairr_image}" compairr --matrix --threads "${n_threads}" --out "${jaccard_matrix_file}" --score Jaccard "${deduplicated_file}" + + else + echo "ERROR: Invalid $analysis_type or $distance provided" + return 1 + fi } diff --git a/apps/compairr/ls6/concatenate_airr_tsv.py b/apps/compairr/ls6/concatenate_airr_tsv.py new file mode 100644 index 0000000..4a8875c --- /dev/null +++ b/apps/compairr/ls6/concatenate_airr_tsv.py @@ -0,0 +1,34 @@ +import json +import argparse +import os +import sys +import gzip + + +if (__name__=="__main__"): + parser = argparse.ArgumentParser(description='Combine airr tsv files for compairr.') + parser.add_argument('-i', '--airr_tsv_files', dest='airr_tsv_files', nargs='*', type=str, help='Repertoire IDs') + parser.add_argument('-o', '--output_path', dest='output_path', type=str, help='Combined airr tsv file name') + args = parser.parse_args() + + if args: + columns = ['repertoire_id', 'sequence_id', 'junction', 'junction_aa', 'v_call', 'j_call', 'duplicate_count'] + all_rows = [] + for path in args.airr_tsv_files: + with gzip.open(path, 'rt') as f: + header = f.readline().strip().split('\t') + # Map columns to their indices + indices = [header.index(col) for col in columns] + for line in f: + parts = line.rstrip('\n').split('\t') + selected = [parts[i] for i in indices] + all_rows.append(selected) + # Build column names once at the end + output_path = args.output_path + # output_path = "all_concatenated_cdr3.tsv" + with open(output_path, 'w') as f: + f.write('\t'.join(columns) + '\n') + for row in all_rows: + f.write('\t'.join(row) + '\n') + + \ No newline at end of file diff --git a/apps/compairr/ls6/tapisjob_app.sh b/apps/compairr/ls6/tapisjob_app.sh index 85ec263..b23f993 100644 --- a/apps/compairr/ls6/tapisjob_app.sh +++ b/apps/compairr/ls6/tapisjob_app.sh @@ -34,9 +34,6 @@ export LAUNCHER_SCHED=interleaved # Start printf "START at $(date)\n\n" -# TODO: how to tell Tapis that the job failed? -export JOB_ERROR=0 - print_parameters print_versions run_compairr_workflow diff --git a/apps/compairr/ls6/test/test-cli.json b/apps/compairr/ls6/test/test-cli.json index fa4acc2..2797759 100644 --- a/apps/compairr/ls6/test/test-cli.json +++ b/apps/compairr/ls6/test/test-cli.json @@ -1,24 +1,31 @@ { - "name": "compairr_test", + "name": "compairr_test_matrix_v1", "appId": "compairr-ls6", - "appVersion": "0.1", + "appVersion": "0.2", "maxMinutes":60, - "execSystemId":"test-bcorrie-airrkb-dev-ls6.tacc.utexas.edu", + "nodeCount": 1, "fileInputs":[ - { "name":"airr_tsv_file", "sourceUrl":"tapis://data-storage.vdjserver.org/irplus/data/TCR/vdjserver1.airr.tsv.gz", "targetPath":"vdjserver1.airr.tsv.gz" } - ], - "parameterSet": { - "schedulerOptions": [ - { "name":"allocation", "arg":"-A MCB23006" } - ], - "containerArgs": [ + ], - "appArgs": [ + "fileInputArrays":[ + { "name":"airr_tsv_files", "sourceUrls": [ + "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_1.airr.tsv.gz", + "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_2.airr.tsv.gz", + "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_3.airr.tsv.gz" + ]} ], - "envVariables": [ - {"key": "analysis_type2", "value": "Jaccard" }, - {"key": "analysis_type", "value": "cluster" }, - {"key": "distance", "value": "1" } - ] + "parameterSet": { + "schedulerOptions": [ + { "name":"allocation", "arg":"-A MCB23006" } + ], + "containerArgs": [ + ], + "appArgs": [ + ], + "envVariables": [ + {"key":"airr_tsv_files", "value": "test_repertoire_1.airr.tsv.gz test_repertoire_2.airr.tsv.gz test_repertoire_3.airr.tsv.gz"}, + {"key": "analysis_type", "value": "matrix" }, + {"key": "distance", "value":"1"} + ] } } diff --git a/apps/compairr/ls6/test/test-cluster.json b/apps/compairr/ls6/test/test-cluster.json new file mode 100644 index 0000000..30c99bf --- /dev/null +++ b/apps/compairr/ls6/test/test-cluster.json @@ -0,0 +1,31 @@ +{ + "name": "compairr_test_cluster", + "appId": "compairr-ls6", + "appVersion": "0.2", + "maxMinutes":60, + "nodeCount": 1, + "fileInputs":[ + + ], + "fileInputArrays":[ + { "name":"airr_tsv_files", "sourceUrls": [ + "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_1.airr.tsv.gz", + "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_2.airr.tsv.gz", + "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_3.airr.tsv.gz" + ]} + ], + "parameterSet": { + "schedulerOptions": [ + { "name":"allocation", "arg":"-A MCB23006" } + ], + "containerArgs": [ + ], + "appArgs": [ + ], + "envVariables": [ + {"key":"airr_tsv_files", "value": "test_repertoire_1.airr.tsv.gz test_repertoire_2.airr.tsv.gz test_repertoire_3.airr.tsv.gz"}, + {"key": "analysis_type", "value": "cluster" }, + {"key": "distance", "value":"1"} + ] + } +} diff --git a/apps/compairr/ls6/test/test-matrix.json b/apps/compairr/ls6/test/test-matrix.json new file mode 100644 index 0000000..018d978 --- /dev/null +++ b/apps/compairr/ls6/test/test-matrix.json @@ -0,0 +1,30 @@ +{ + "name": "compairr_test_matrix", + "appId": "compairr-ls6", + "appVersion": "0.2", + "maxMinutes":60, + "nodeCount": 1, + "fileInputs":[ + + ], + "fileInputArrays":[ + { "name":"airr_tsv_files", "sourceUrls": [ + "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_1.airr.tsv.gz", + "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_2.airr.tsv.gz", + "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_3.airr.tsv.gz" + ]} + ], + "parameterSet": { + "schedulerOptions": [ + { "name":"allocation", "arg":"-A MCB23006" } + ], + "containerArgs": [ + ], + "appArgs": [ + ], + "envVariables": [ + {"key":"airr_tsv_files", "value": "test_repertoire_1.airr.tsv.gz test_repertoire_2.airr.tsv.gz test_repertoire_3.airr.tsv.gz"}, + {"key": "analysis_type", "value": "matrix" } + ] + } +} diff --git a/apps/compairr/ls6/test/test-overlap.json b/apps/compairr/ls6/test/test-overlap.json new file mode 100644 index 0000000..0c77fb2 --- /dev/null +++ b/apps/compairr/ls6/test/test-overlap.json @@ -0,0 +1,31 @@ +{ + "name": "compairr_test_overlap", + "appId": "compairr-ls6", + "appVersion": "0.2", + "maxMinutes":60, + "nodeCount": 1, + "fileInputs":[ + + ], + "fileInputArrays":[ + { "name":"airr_tsv_files", "sourceUrls": [ + "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_1.airr.tsv.gz", + "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_2.airr.tsv.gz", + "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_3.airr.tsv.gz" + ]} + ], + "parameterSet": { + "schedulerOptions": [ + { "name":"allocation", "arg":"-A MCB23006" } + ], + "containerArgs": [ + ], + "appArgs": [ + ], + "envVariables": [ + {"key":"airr_tsv_files", "value": "test_repertoire_1.airr.tsv.gz test_repertoire_2.airr.tsv.gz test_repertoire_3.airr.tsv.gz"}, + {"key": "analysis_type", "value": "overlap" }, + {"key": "distance", "value":"1"} + ] + } +} diff --git a/data/test/test_repertoire_1.airr.tsv.gz b/data/test/test_repertoire_1.airr.tsv.gz new file mode 100644 index 0000000..0158d27 Binary files /dev/null and b/data/test/test_repertoire_1.airr.tsv.gz differ diff --git a/data/test/test_repertoire_2.airr.tsv.gz b/data/test/test_repertoire_2.airr.tsv.gz new file mode 100644 index 0000000..097cda5 Binary files /dev/null and b/data/test/test_repertoire_2.airr.tsv.gz differ diff --git a/data/test/test_repertoire_3.airr.tsv.gz b/data/test/test_repertoire_3.airr.tsv.gz new file mode 100644 index 0000000..e69a595 Binary files /dev/null and b/data/test/test_repertoire_3.airr.tsv.gz differ