Added compairr app for 3 types of analysis.

apps/compairr/ls6/Makefile

@@ -1,7 +1,7 @@
 # helper commands for tapis v3 apps
 
 APP_NAME=compairr
-APP_VERSION=0.1
+APP_VERSION=0.2
 APP_SYSTEM=ls6
 
 APP_ID=${APP_NAME}-${APP_SYSTEM}
@@ -30,13 +30,14 @@ help:
 	@echo ""
 	@echo "Testing:"
 	@echo "make test-job  -- Submit test job"
+	@echo "make test-all  -- Submit all test jobs"
 	@echo ""
 
 create:
-	tapis_apps_create ${APP_JSON}
+	vdjserver-tools apps create ${APP_JSON}
 
 update:
-	tapis_apps_update ${APP_ID} ${APP_VERSION} ${APP_JSON}
+	vdjserver-tools apps update ${APP_ID} ${APP_VERSION} ${APP_JSON}
 
 clean:
 	rm -rf ${BUILD_DIR} ${APP_ARCHIVE}
@@ -50,15 +51,19 @@ common-scripts:
 build: clean common-scripts
 	cp ${APP_JSON} ${BUILD_DIR}
 	cp tapisjob_app.sh ${BUILD_DIR}
+	cp concatenate_airr_tsv.py ${BUILD_DIR}
 	cp ${APP_NAME}_common.sh ${BUILD_DIR}
 	tar zcf ${APP_ARCHIVE} -C ${BUILD_DIR} .
 
 deploy: build
-	tapis_files_upload /akc/apps/${APP_ARCHIVE} ${APP_ARCHIVE}
+	vdjserver-tools files upload ${APP_ARCHIVE} /apps/${APP_ARCHIVE}
 
 test-job:
-	tapis_jobs_submit test/test-cli.json
+	vdjserver-tools jobs submit test/test-cli.json
 
-test-cache:
-	tapis_jobs_submit test/test-cache.json
+test-all:
+	vdjserver-tools jobs submit test/test-cli.json
+	vdjserver-tools jobs submit test/test-cluster.json
+	vdjserver-tools jobs submit test/test-matrix.json
+	vdjserver-tools jobs submit test/test-overlap.json
 

apps/compairr/ls6/compairr-ls6.json

@@ -1,16 +1,19 @@
 {
     "id":"compairr-ls6",
-    "version":"0.1",
+    "version":"0.2",
     "description":"CompAIRR analysis for AKC",
     "runtime":"ZIP",
     "runtimeOptions": ["NONE"],
-    "containerImage":"tapis://data-storage.vdjserver.org/akc/apps/compairr-ls6-0.1.tgz",
+    "containerImage":"tapis://data-storage.vdjserver.org/apps/compairr-ls6-0.2.tgz",
     "jobType":"BATCH",
     "jobAttributes": {
         "description": "default job description",
+        "execSystemId": "test-ls6.tacc.utexas.edu",
         "fileInputs":[
-          { "name":"compairr_image", "inputMode": "FIXED", "envKey":"compairr_image", "sourceUrl":"tapis://data-storage.vdjserver.org/singularity/ir_compairr-1.13.0.sif", "targetPath":"ir_compairr-1.13.0.sif" },
-          { "name":"airr_tsv_file", "inputMode": "REQUIRED", "envKey":"airr_tsv_file", "targetPath":"." }
+          { "name":"compairr_image", "inputMode": "FIXED", "envKey":"compairr_image", "sourceUrl":"tapis://data-storage.vdjserver.org/singularity/ir_compairr-1.13.0.sif", "targetPath":"ir_compairr-1.13.0.sif" }
+        ],
+        "fileInputArrays":[
+          { "name":"airr_tsv_files", "inputMode":"REQUIRED", "targetDir":"."}
         ],
         "parameterSet": {
             "logConfig": {
@@ -22,9 +25,9 @@
             "appArgs": [
             ],
             "envVariables": [
-                {"key": "analysis_type", "inputMode": "REQUIRED", "value": "product" },
-                {"key": "distance", "inputMode": "REQUIRED", "value": "1" },
-                {"key": "file_type", "inputMode": "REQUIRED", "value": "rearrangement" }
+                {"key": "airr_tsv_files", "inputMode": "REQUIRED"},
+                {"key": "analysis_type", "inputMode": "REQUIRED"},
+                {"key": "distance", "inputMode": "INCLUDE_BY_DEFAULT"}
             ]
         }
     }

apps/compairr/ls6/compairr_common.sh

@@ -34,49 +34,58 @@ function print_versions() {
 function print_parameters() {
     echo "Input files:"
     echo "compairr_image=${compairr_image}"
-    echo "airr_tsv_file=${airr_tsv_file}"
+    echo "airr_tsv_files=${airr_tsv_files}"
     echo ""
     echo "Application parameters:"
     echo "analysis_type=${analysis_type}"
     echo "distance=${distance}"
-    echo "file_type=${file_type}"
 }
 
 function run_compairr_workflow() {
     initProvenance
+    #concatenate files
+    concatenated_file="concat.tsv"
+    ${PYTHON} concatenate_airr_tsv.py -i $airr_tsv_files -o $concatenated_file
+    #deduplicate files
+    deduplicated_file="dedup_${concatenated_file}"
+    echo "Command: apptainer exec -e ${compairr_image} compairr --deduplicate --out ${deduplicated_file}"
+    apptainer exec -e "${compairr_image}" compairr --deduplicate --out "${deduplicated_file}" "${concatenated_file}"
 
-    # expand rearrangement file if its compressed
-    expandfile $airr_tsv_file
-    #noArchive $file
-
-    # Assuming airr.tsv extension
-    fileBasename="${file%.*}" # file.airr.tsv -> file.airr
-    fileBasename="${fileBasename%.*}" # file.airr -> file
-
-    # Run compairr in matrix or cluster mode, using the "analysis_type" to determine
-    # which method to use. $distance is used only in cluster mode.
-    if [[ "$file_type" == "rearrangement" ]] ; then
-        if [[ "$analysis_type" == "cluster" ]] ; then
-	    if [[ ! "x$distance"  == "x" ]]; then
-	        re='^[0-9]+$'
-                if [[ $distance =~ $re ]]; then
-	            echo "Runnig: apptainer exec -e ${compairr_image} compairr -f -e -u --cluster ${file} -d ${distance} --out $fileBasename.cluster.tsv"
-	            apptainer exec -e ${compairr_image} compairr -f -e -u --cluster ${file} -d ${distance} --out $fileBasename.cluster.tsv
-		else
-                    echo "ERROR: Distance metric ($distance) integer and greater than 0 required"
-		    return
-		fi
-	    else
-		echo "ERROR: Distance metric not provided"
-		return
-	    fi
-        elif [[ "$analysis_type" == "product" || "$analysis_type" == "MH" || "$analysis_type" == "Morisita-Horn" ]] ; then
-	    echo "Runnig: apptainer exec -e ${compairr_image} compairr -f -e -u -s $analysis_type --matrix ${file} --out $fileBasename.matrix.tsv"
-	    apptainer exec -e ${compairr_image} compairr -f -e -u -s $analysis_type --matrix ${file} --out $fileBasename.matrix.tsv
-        else
-	    echo "ERROR: Invalid analysis type $analysis_type provided"
-	    return
-	fi
-    fi
+    # Assuming .tsv extension
+    file_basename="${deduplicated_file%.*}" # file.tsv -> file
+    n_threads=20
+
+    cluster_file="${file_basename}_d_${distance}_clust.tsv"
+
+    default_matrix_file="${file_basename}_d_${distance}_prodmat.txt"
+    pairs_file="${file_basename}_d_${distance}_pairs.tsv"
+
+    mh_matrix_file="${file_basename}_MHmat.txt"
+    jaccard_matrix_file="${file_basename}_Jacmat.txt"
+
+    if [[ "$analysis_type" == "cluster" ]] ; then
+        echo "Running Cluster Analysis for distance $distance"
+
+        echo "Command: apptainer exec -e ${compairr_image} compairr --cluster ${deduplicated_file} -d ${distance} --out $cluster_file"
+        apptainer exec -e "${compairr_image}" compairr --cluster -d "${distance}" --threads "${n_threads}" --out "$cluster_file" "${deduplicated_file}"
+
+    elif [[ "$analysis_type" == "overlap" ]]; then
+        echo "Calculating overlap analysis for distance $distance."
 
+        echo "Command: apptainer exec -e ${compairr_image} compairr --matrix -d ${distance} --pairs ${pairs_file} ${deduplicated_file}"
+        apptainer exec -e "${compairr_image}" compairr --matrix -d "${distance}" --threads "${n_threads}" --pairs "${pairs_file}" "${deduplicated_file}"
+
+    elif [[ "$analysis_type" == "matrix" ]]; then
+        echo "Running matrix analysis for MH and Jaccard Score."
+
+        echo "Command: apptainer -e ${compairr_image} compairr --matrix --out ${mh_matrix_file} --score MH ${deduplicated_file}"
+        apptainer exec -e "${compairr_image}" compairr --matrix --threads "${n_threads}" --out "${mh_matrix_file}" --score MH "${deduplicated_file}"
+
+        echo "Command: apptainer exec -e ${compairr_image} compairr --matrix --out ${jaccard_matrix_file} --score Jaccard ${deduplicated_file} "
+        apptainer exec -e "${compairr_image}" compairr --matrix --threads "${n_threads}" --out "${jaccard_matrix_file}" --score Jaccard "${deduplicated_file}"
+
+    else
+        echo "ERROR: Invalid $analysis_type or $distance provided"
+        return 1
+    fi
 }

apps/compairr/ls6/concatenate_airr_tsv.py

@@ -0,0 +1,34 @@
+import json
+import argparse
+import os
+import sys
+import gzip
+
+
+if (__name__=="__main__"):
+    parser = argparse.ArgumentParser(description='Combine airr tsv files for compairr.')
+    parser.add_argument('-i', '--airr_tsv_files', dest='airr_tsv_files', nargs='*', type=str, help='Repertoire IDs')
+    parser.add_argument('-o', '--output_path', dest='output_path', type=str, help='Combined airr tsv file name')
+    args = parser.parse_args()
+
+    if args:
+        columns = ['repertoire_id', 'sequence_id', 'junction', 'junction_aa', 'v_call', 'j_call', 'duplicate_count']
+        all_rows = []
+        for path in args.airr_tsv_files:
+            with gzip.open(path, 'rt') as f:
+                header = f.readline().strip().split('\t')
+                # Map columns to their indices
+                indices = [header.index(col) for col in columns]
+                for line in f:
+                    parts = line.rstrip('\n').split('\t')
+                    selected = [parts[i] for i in indices]
+                    all_rows.append(selected)
+        # Build column names once at the end
+        output_path = args.output_path
+        # output_path = "all_concatenated_cdr3.tsv"
+        with open(output_path, 'w') as f:
+            f.write('\t'.join(columns) + '\n')
+            for row in all_rows:
+                f.write('\t'.join(row) + '\n')
+
+

apps/compairr/ls6/tapisjob_app.sh

@@ -34,9 +34,6 @@ export LAUNCHER_SCHED=interleaved
 # Start
 printf "START at $(date)\n\n"
 
-# TODO: how to tell Tapis that the job failed?
-export JOB_ERROR=0
-
 print_parameters
 print_versions
 run_compairr_workflow

apps/compairr/ls6/test/test-cli.json

@@ -1,24 +1,31 @@
 {
-    "name": "compairr_test",
+    "name": "compairr_test_matrix_v1",
     "appId": "compairr-ls6",
-    "appVersion": "0.1",
+    "appVersion": "0.2",
     "maxMinutes":60,
-    "execSystemId":"test-bcorrie-airrkb-dev-ls6.tacc.utexas.edu",
+    "nodeCount": 1,
     "fileInputs":[
-      { "name":"airr_tsv_file", "sourceUrl":"tapis://data-storage.vdjserver.org/irplus/data/TCR/vdjserver1.airr.tsv.gz", "targetPath":"vdjserver1.airr.tsv.gz" }
-    ],
-    "parameterSet": {
-        "schedulerOptions": [
-            { "name":"allocation", "arg":"-A MCB23006" }
-        ],
-        "containerArgs": [
+
         ],
-        "appArgs": [
+        "fileInputArrays":[
+          { "name":"airr_tsv_files", "sourceUrls": [
+            "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_1.airr.tsv.gz", 
+            "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_2.airr.tsv.gz",
+            "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_3.airr.tsv.gz"
+            ]}
         ],
-        "envVariables": [
-            {"key": "analysis_type2", "value": "Jaccard" },
-            {"key": "analysis_type", "value": "cluster" },
-            {"key": "distance", "value": "1" }
-        ]
+        "parameterSet": {
+            "schedulerOptions": [
+            { "name":"allocation", "arg":"-A MCB23006" }
+            ],
+            "containerArgs": [
+            ],
+            "appArgs": [
+            ],
+            "envVariables": [
+                {"key":"airr_tsv_files", "value": "test_repertoire_1.airr.tsv.gz test_repertoire_2.airr.tsv.gz test_repertoire_3.airr.tsv.gz"},
+                {"key": "analysis_type", "value": "matrix" },
+                {"key": "distance", "value":"1"}
+            ]
     }
 }

apps/compairr/ls6/test/test-cluster.json

@@ -0,0 +1,31 @@
+{
+    "name": "compairr_test_cluster",
+    "appId": "compairr-ls6",
+    "appVersion": "0.2",
+    "maxMinutes":60,
+    "nodeCount": 1,
+    "fileInputs":[
+
+        ],
+        "fileInputArrays":[
+          { "name":"airr_tsv_files", "sourceUrls": [
+            "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_1.airr.tsv.gz", 
+            "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_2.airr.tsv.gz",
+            "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_3.airr.tsv.gz"
+            ]}
+        ],
+        "parameterSet": {
+            "schedulerOptions": [
+            { "name":"allocation", "arg":"-A MCB23006" }
+            ],
+            "containerArgs": [
+            ],
+            "appArgs": [
+            ],
+            "envVariables": [
+                {"key":"airr_tsv_files", "value": "test_repertoire_1.airr.tsv.gz test_repertoire_2.airr.tsv.gz test_repertoire_3.airr.tsv.gz"},
+                {"key": "analysis_type", "value": "cluster" },
+                {"key": "distance", "value":"1"}
+            ]
+    }
+}

apps/compairr/ls6/test/test-matrix.json

@@ -0,0 +1,30 @@
+{
+    "name": "compairr_test_matrix",
+    "appId": "compairr-ls6",
+    "appVersion": "0.2",
+    "maxMinutes":60,
+    "nodeCount": 1,
+    "fileInputs":[
+
+        ],
+        "fileInputArrays":[
+          { "name":"airr_tsv_files", "sourceUrls": [
+            "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_1.airr.tsv.gz", 
+            "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_2.airr.tsv.gz",
+            "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_3.airr.tsv.gz"
+            ]}
+        ],
+        "parameterSet": {
+            "schedulerOptions": [
+            { "name":"allocation", "arg":"-A MCB23006" }
+            ],
+            "containerArgs": [
+            ],
+            "appArgs": [
+            ],
+            "envVariables": [
+                {"key":"airr_tsv_files", "value": "test_repertoire_1.airr.tsv.gz test_repertoire_2.airr.tsv.gz test_repertoire_3.airr.tsv.gz"},
+                {"key": "analysis_type", "value": "matrix" }
+            ]
+    }
+}

apps/compairr/ls6/test/test-overlap.json

@@ -0,0 +1,31 @@
+{
+    "name": "compairr_test_overlap",
+    "appId": "compairr-ls6",
+    "appVersion": "0.2",
+    "maxMinutes":60,
+    "nodeCount": 1,
+    "fileInputs":[
+
+        ],
+        "fileInputArrays":[
+          { "name":"airr_tsv_files", "sourceUrls": [
+            "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_1.airr.tsv.gz", 
+            "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_2.airr.tsv.gz",
+            "tapis://data-storage.vdjserver.org/apps/data/test/test_repertoire_3.airr.tsv.gz"
+            ]}
+        ],
+        "parameterSet": {
+            "schedulerOptions": [
+            { "name":"allocation", "arg":"-A MCB23006" }
+            ],
+            "containerArgs": [
+            ],
+            "appArgs": [
+            ],
+            "envVariables": [
+                {"key":"airr_tsv_files", "value": "test_repertoire_1.airr.tsv.gz test_repertoire_2.airr.tsv.gz test_repertoire_3.airr.tsv.gz"},
+                {"key": "analysis_type", "value": "overlap" },
+                {"key": "distance", "value":"1"}
+            ]
+    }
+}