From 09aabc5738d88698a8b06274b5107bc757595da7 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 9 Jun 2026 08:33:42 -0700 Subject: [PATCH 01/18] Hack to allow maintenance jobs to respect cancellation --- .../SequenceAnalysisMaintenanceTask.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java index da250d486..e77e971b6 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java @@ -8,6 +8,8 @@ import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.Sort; import org.labkey.api.data.TableInfo; @@ -16,7 +18,9 @@ import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.ldk.LDKService; +import org.labkey.api.pipeline.CancelledException; import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.PipelineStatusFile; @@ -74,6 +78,26 @@ public String getName() return "DeleteSequenceAnalysisArtifacts"; } + private void checkJobCancelled(Logger log) + { + // Make the assumption there is only one active maintenance job at a time: + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("description"), "System Maintenance"). + addCondition(FieldKey.fromString("Folder"), ContainerManager.getRoot().getId()). + addInClause(FieldKey.fromString("Status"), Arrays.asList(PipelineJob.TaskStatus.cancelling.name(), PipelineJob.TaskStatus.running.name())); + Integer rowId = new TableSelector(DbSchema.get("pipeline", DbSchemaType.Module).getTable("Pob"), PageFlowUtil.set("RowId"), filter, null).getObject(Integer.class); + if (rowId == null) + { + log.error("Unable to find rowId for job"); + return; + } + + PipelineStatusFile sf = PipelineService.get().getStatusFile(rowId); + if (sf.getStatus() == PipelineJob.TaskStatus.cancelling.name()) + { + throw new CancelledException(); + } + } + @Override public void run(Logger log) { @@ -158,6 +182,7 @@ private void verifySequenceDataPresent(Logger log) if (i % 1000 == 0) { log.info("readdata " + i + " of " + readDatas.size() + ". Current container: " + ContainerManager.getForId(rd.getContainer()).getPath()); + checkJobCancelled(log); } if (rd.getFileId1() != null) @@ -221,6 +246,7 @@ else if (!d.getFile().exists()) if (i % 1000 == 0) { log.info("analysis " + i + " of " + analyses.size() + ". Current container: " + ContainerManager.getForId(m.getContainer()).getPath()); + checkJobCancelled(log); } if (m.getAlignmentFile() != null) @@ -296,7 +322,11 @@ else if (sf.getFilePath() == null) private void processContainer(Container c, Logger log) throws IOException, PipelineJobException { if (!c.isWorkbook()) + { log.info("processing container: " + c.getPath()); + } + + checkJobCancelled(log); PipeRoot root = PipelineService.get().getPipelineRootSetting(c); if (root != null && !root.isCloudRoot()) From 359a9474fb016ffbe593c6554671f4e27de093ed Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 9 Jun 2026 09:31:49 -0700 Subject: [PATCH 02/18] Use constants --- .../sequenceanalysis/SequenceAnalysisMaintenanceTask.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java index e77e971b6..6cd2ca380 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java @@ -61,6 +61,9 @@ */ public class SequenceAnalysisMaintenanceTask implements MaintenanceTask { + private static final String SYSTEM_MAINTENANCE_DESCRIPTION = "System Maintenance"; + private static final String JOB_TABLE = "Job"; + public SequenceAnalysisMaintenanceTask() { @@ -78,13 +81,14 @@ public String getName() return "DeleteSequenceAnalysisArtifacts"; } + // NOTE: if there is a more direct way to locate the JobID this should be replaced private void checkJobCancelled(Logger log) { // Make the assumption there is only one active maintenance job at a time: - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("description"), "System Maintenance"). + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("description"), SYSTEM_MAINTENANCE_DESCRIPTION). addCondition(FieldKey.fromString("Folder"), ContainerManager.getRoot().getId()). addInClause(FieldKey.fromString("Status"), Arrays.asList(PipelineJob.TaskStatus.cancelling.name(), PipelineJob.TaskStatus.running.name())); - Integer rowId = new TableSelector(DbSchema.get("pipeline", DbSchemaType.Module).getTable("Pob"), PageFlowUtil.set("RowId"), filter, null).getObject(Integer.class); + Integer rowId = new TableSelector(DbSchema.get("pipeline", DbSchemaType.Module).getTable(JOB_TABLE), PageFlowUtil.set("RowId"), filter, null).getObject(Integer.class); if (rowId == null) { log.error("Unable to find rowId for job"); From 626848772c4c82960dccd521e1d088cc3d817629 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 9 Jun 2026 11:31:17 -0700 Subject: [PATCH 03/18] Fix typo --- .../sequenceanalysis/SequenceAnalysisMaintenanceTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java index 6cd2ca380..1310b48b3 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java @@ -62,7 +62,7 @@ public class SequenceAnalysisMaintenanceTask implements MaintenanceTask { private static final String SYSTEM_MAINTENANCE_DESCRIPTION = "System Maintenance"; - private static final String JOB_TABLE = "Job"; + private static final String JOB_TABLE = "statusfiles"; public SequenceAnalysisMaintenanceTask() { @@ -96,7 +96,7 @@ private void checkJobCancelled(Logger log) } PipelineStatusFile sf = PipelineService.get().getStatusFile(rowId); - if (sf.getStatus() == PipelineJob.TaskStatus.cancelling.name()) + if (PipelineJob.TaskStatus.cancelling.name().equalsIgnoreCase(sf.getStatus())) { throw new CancelledException(); } From 6d9845930f1a4b27c1f6d9e92b1c5fb05f0ddc36 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 9 Jun 2026 12:47:45 -0700 Subject: [PATCH 04/18] Also track TRD data --- singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R | 1 + 1 file changed, 1 insertion(+) diff --git a/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R b/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R index 46bdb584f..ee5de2b82 100644 --- a/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R +++ b/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R @@ -17,6 +17,7 @@ for (datasetId in names(seuratObjects)) { } Rdiscvr::IdentifyAndStoreActiveClonotypes(seuratObj, chain = 'TRA', storeStimLevelData = FALSE, minEDS = minEDS) + Rdiscvr::IdentifyAndStoreActiveClonotypes(seuratObj, chain = 'TRD', storeStimLevelData = FALSE, minEDS = minEDS) Rdiscvr::IdentifyAndStoreActiveClonotypes(seuratObj, chain = 'TRB', minEDS = minEDS) saveData(seuratObj, datasetId) From 3013b74e6a3406a2c6e3ed655ba70123e1f275aa Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 9 Jun 2026 13:56:17 -0700 Subject: [PATCH 05/18] CSP exceptions --- .../sequenceanalysis/SequenceAnalysisMaintenanceTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java index 1310b48b3..bbe2d3731 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java @@ -86,8 +86,8 @@ private void checkJobCancelled(Logger log) { // Make the assumption there is only one active maintenance job at a time: SimpleFilter filter = new SimpleFilter(FieldKey.fromString("description"), SYSTEM_MAINTENANCE_DESCRIPTION). - addCondition(FieldKey.fromString("Folder"), ContainerManager.getRoot().getId()). - addInClause(FieldKey.fromString("Status"), Arrays.asList(PipelineJob.TaskStatus.cancelling.name(), PipelineJob.TaskStatus.running.name())); + addCondition(FieldKey.fromString("container"), ContainerManager.getRoot().getId()). + addInClause(FieldKey.fromString("status"), Arrays.asList(PipelineJob.TaskStatus.cancelling.name(), PipelineJob.TaskStatus.running.name())); Integer rowId = new TableSelector(DbSchema.get("pipeline", DbSchemaType.Module).getTable(JOB_TABLE), PageFlowUtil.set("RowId"), filter, null).getObject(Integer.class); if (rowId == null) { From 5df002665f02ce5cb2baa7fdb0082acbd091fed4 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 9 Jun 2026 20:49:42 -0700 Subject: [PATCH 06/18] Bugfix to maintenance job cancellation --- .../SequenceAnalysisMaintenanceTask.java | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java index bbe2d3731..2175d2431 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java @@ -44,6 +44,7 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -81,17 +82,37 @@ public String getName() return "DeleteSequenceAnalysisArtifacts"; } - // NOTE: if there is a more direct way to locate the JobID this should be replaced + // NOTE: if there is a more direct way to locate the JobID this hack should be replaced private void checkJobCancelled(Logger log) { // Make the assumption there is only one active maintenance job at a time: SimpleFilter filter = new SimpleFilter(FieldKey.fromString("description"), SYSTEM_MAINTENANCE_DESCRIPTION). addCondition(FieldKey.fromString("container"), ContainerManager.getRoot().getId()). - addInClause(FieldKey.fromString("status"), Arrays.asList(PipelineJob.TaskStatus.cancelling.name(), PipelineJob.TaskStatus.running.name())); - Integer rowId = new TableSelector(DbSchema.get("pipeline", DbSchemaType.Module).getTable(JOB_TABLE), PageFlowUtil.set("RowId"), filter, null).getObject(Integer.class); - if (rowId == null) + addCondition(FieldKey.fromString("modified"), new Date(), CompareType.DATE_EQUAL); + int rowId = new TableSelector(DbSchema.get("pipeline", DbSchemaType.Module).getTable(JOB_TABLE), PageFlowUtil.set("RowId", "Status"), filter, null).resultsStream(false).filter(rs -> { + try + { + String val = rs.getString(FieldKey.fromString("status")); + return val != null && (val.toLowerCase().startsWith(PipelineJob.TaskStatus.cancelling.name()) || val.toLowerCase().startsWith(PipelineJob.TaskStatus.running.name())); + } + catch (SQLException e) + { + throw new RuntimeException(e); + } + }).map(rs -> { + try + { + return rs.getInt(FieldKey.fromString("RowId")); + } + catch (SQLException e) + { + throw new RuntimeException(e); + } + }).max(Integer::compareTo).orElse(-1); + + if (rowId == -1) { - log.error("Unable to find rowId for job"); + log.debug("Unable to find rowId for job", new Exception()); return; } From 5b4f78cf212d3f8aac9ce94bd6689d09437c47c6 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 10 Jun 2026 06:54:11 -0700 Subject: [PATCH 07/18] Bugfix to maintenance job cancellation --- .../SequenceAnalysisMaintenanceTask.java | 27 ++++--------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java index 2175d2431..1c9c469a0 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java @@ -44,7 +44,6 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -89,30 +88,14 @@ private void checkJobCancelled(Logger log) SimpleFilter filter = new SimpleFilter(FieldKey.fromString("description"), SYSTEM_MAINTENANCE_DESCRIPTION). addCondition(FieldKey.fromString("container"), ContainerManager.getRoot().getId()). addCondition(FieldKey.fromString("modified"), new Date(), CompareType.DATE_EQUAL); - int rowId = new TableSelector(DbSchema.get("pipeline", DbSchemaType.Module).getTable(JOB_TABLE), PageFlowUtil.set("RowId", "Status"), filter, null).resultsStream(false).filter(rs -> { - try - { - String val = rs.getString(FieldKey.fromString("status")); - return val != null && (val.toLowerCase().startsWith(PipelineJob.TaskStatus.cancelling.name()) || val.toLowerCase().startsWith(PipelineJob.TaskStatus.running.name())); - } - catch (SQLException e) - { - throw new RuntimeException(e); - } - }).map(rs -> { - try - { - return rs.getInt(FieldKey.fromString("RowId")); - } - catch (SQLException e) - { - throw new RuntimeException(e); - } - }).max(Integer::compareTo).orElse(-1); + int rowId = new TableSelector(DbSchema.get("pipeline", DbSchemaType.Module).getTable(JOB_TABLE), PageFlowUtil.set("RowId", "Status"), filter, null).getMapCollection().stream().filter(map -> { + String val = String.valueOf(map.get("status")); + return val != null && (val.toLowerCase().startsWith(PipelineJob.TaskStatus.cancelling.name()) || val.toLowerCase().startsWith(PipelineJob.TaskStatus.running.name())); + }).map(rs -> Integer.parseInt(String.valueOf(rs.get("rowid")))).max(Integer::compareTo).orElse(-1); if (rowId == -1) { - log.debug("Unable to find rowId for job", new Exception()); + log.warn("Unable to find rowId for job", new Exception()); return; } From 6eb8ab3e5a781e70a4025dc5aa62b4b1a3f6dc48 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Jun 2026 12:52:56 -0700 Subject: [PATCH 08/18] Expose filterDisallowedClasses for RIRA --- singlecell/resources/chunks/RunRiraClassification.R | 2 +- .../singlecell/pipeline/singlecell/RunRiraClassification.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/singlecell/resources/chunks/RunRiraClassification.R b/singlecell/resources/chunks/RunRiraClassification.R index 89740793e..eeded9f15 100644 --- a/singlecell/resources/chunks/RunRiraClassification.R +++ b/singlecell/resources/chunks/RunRiraClassification.R @@ -2,7 +2,7 @@ for (datasetId in names(seuratObjects)) { printName(datasetId) seuratObj <- readSeuratRDS(seuratObjects[[datasetId]]) - seuratObj <- RIRA::Classify_ImmuneCells(seuratObj, maxBatchSize = maxBatchSize, retainProbabilityMatrix = retainProbabilityMatrix, maxAllowedUnknown = maxAllowedUnknown) + seuratObj <- RIRA::Classify_ImmuneCells(seuratObj, maxBatchSize = maxBatchSize, retainProbabilityMatrix = retainProbabilityMatrix, maxAllowedUnknown = maxAllowedUnknown, filterDisallowedClasses = filterDisallowedClasses) seuratObj <- RIRA::Classify_TNK(seuratObj, maxBatchSize = maxBatchSize, retainProbabilityMatrix = retainProbabilityMatrix) seuratObj <- RIRA::Classify_Myeloid(seuratObj, maxBatchSize = maxBatchSize, retainProbabilityMatrix = retainProbabilityMatrix) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunRiraClassification.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunRiraClassification.java index 9ee51c7b0..fa518d0b7 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunRiraClassification.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunRiraClassification.java @@ -28,6 +28,10 @@ public Provider() {{ put("checked", true); }}, true), + SeuratToolParameter.create("filterDisallowedClasses", "Filter Disallowed Classes", "If true, then cells will be filtered based on expression of common contaminants, such as RBC or platelet genes ", "checkbox", new JSONObject() + {{ + put("checked", true); + }}, true), SeuratToolParameter.create("maxAllowedUnknown", "Max Allowed Unknown", "If provided, this step will throw an error is more than this fraction of cells fail the check for disallowed UCell combinations", "ldk-numberfield", new JSONObject() {{ put("minValue", 0); From f68174d5351b0827ea3d608aa326c4d5138cce55 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Jun 2026 12:55:18 -0700 Subject: [PATCH 09/18] Expose filterDisallowedClasses for UpdateSeuratPrototype --- singlecell/resources/chunks/UpdateSeuratPrototype.R | 2 +- .../singlecell/pipeline/singlecell/UpdateSeuratPrototype.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/singlecell/resources/chunks/UpdateSeuratPrototype.R b/singlecell/resources/chunks/UpdateSeuratPrototype.R index 56e48e7d5..2da10f1ac 100644 --- a/singlecell/resources/chunks/UpdateSeuratPrototype.R +++ b/singlecell/resources/chunks/UpdateSeuratPrototype.R @@ -26,7 +26,7 @@ for (datasetId in names(seuratObjects)) { } if (runRira) { - seuratObj <- RIRA::Classify_ImmuneCells(seuratObj, maxBatchSize = 500000, retainProbabilityMatrix = FALSE, maxAllowedUnknown = maxAllowedUnknown) + seuratObj <- RIRA::Classify_ImmuneCells(seuratObj, maxBatchSize = 500000, retainProbabilityMatrix = FALSE, maxAllowedUnknown = maxAllowedUnknown, filterDisallowedClasses = filterDisallowedClasses) seuratObj <- RIRA::Classify_TNK(seuratObj, maxBatchSize = 500000, retainProbabilityMatrix = FALSE) seuratObj <- RIRA::Classify_Myeloid(seuratObj, maxBatchSize = 500000, retainProbabilityMatrix = FALSE) } diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/UpdateSeuratPrototype.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/UpdateSeuratPrototype.java index 2518b0d1a..63d597ef8 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/UpdateSeuratPrototype.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/UpdateSeuratPrototype.java @@ -64,6 +64,10 @@ public Provider() {{ put("checked", false); }}, false), + SeuratToolParameter.create("filterDisallowedClasses", "Filter Disallowed Classes (RIRA)", "This applies to RIRA classification. If true, then cells will be filtered based on expression of common contaminants, such as RBC or platelet genes ", "checkbox", new JSONObject() + {{ + put("checked", true); + }}, true), SeuratToolParameter.create("maxAllowedUnknown", "Max Allowed Unknown (RIRA)", "If provided, this step will throw an error is more than this fraction of cells fail the check for disallowed UCell combinations", "ldk-numberfield", new JSONObject() {{ put("minValue", 0); From c8ca6a7d64472a11f0f2d49afd23b2fc3da878b5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 15 Jun 2026 09:34:47 -0700 Subject: [PATCH 10/18] Add message for Exception --- .../sequenceanalysis/SequenceAnalysisMaintenanceTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java index 1c9c469a0..82e76b851 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java @@ -95,7 +95,7 @@ private void checkJobCancelled(Logger log) if (rowId == -1) { - log.warn("Unable to find rowId for job", new Exception()); + log.warn("Unable to find rowId for job", new Exception("Unable to find rowId for job")); return; } From dbd53105f9947bd6a240616fb1f836b89262b441 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 21 Jun 2026 10:48:16 -0700 Subject: [PATCH 11/18] Add wrapper for Kracken2 (#401) * Add wrapper for Kracken2 --- .../SequenceAnalysisModule.java | 3 +- .../run/preprocessing/Kraken2Step.java | 195 ++++++++++++++++++ .../external/labModules/SequenceTest.java | 5 +- 3 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java index 93688d270..4442068da 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java @@ -152,6 +152,7 @@ import org.labkey.sequenceanalysis.run.preprocessing.FastqcProcessingStep; import org.labkey.sequenceanalysis.run.preprocessing.FilterReadsStep; import org.labkey.sequenceanalysis.run.preprocessing.FlashPipelineStep; +import org.labkey.sequenceanalysis.run.preprocessing.Kraken2Step; import org.labkey.sequenceanalysis.run.preprocessing.PrintReadsContainingStep; import org.labkey.sequenceanalysis.run.preprocessing.TagPcrSummaryStep; import org.labkey.sequenceanalysis.run.preprocessing.TrimmomaticWrapper; @@ -291,7 +292,7 @@ public static void registerPipelineSteps() SequencePipelineService.get().registerPipelineStep(new CutadaptWrapper.Provider()); SequencePipelineService.get().registerPipelineStep(new FastqcProcessingStep.Provider()); SequencePipelineService.get().registerPipelineStep(new CutadaptCropWrapper.Provider()); - //SequencePipelineService.get().registerPipelineStep(new BlastFilterPipelineStep.Provider()); + SequencePipelineService.get().registerPipelineStep(new Kraken2Step.Provider()); //ref library SequencePipelineService.get().registerPipelineStep(new DNAReferenceLibraryStep.Provider()); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java new file mode 100644 index 000000000..6f22f595f --- /dev/null +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java @@ -0,0 +1,195 @@ +package org.labkey.sequenceanalysis.run.preprocessing; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.PipelineJobService; +import org.labkey.api.sequenceanalysis.SequenceAnalysisService; +import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.CommandLineParam; +import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; +import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.PreprocessingStep; +import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; +import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.api.sequenceanalysis.run.AbstractCommandPipelineStep; +import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; +import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Pair; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class Kraken2Step extends AbstractCommandPipelineStep implements PreprocessingStep +{ + private static final String DB_PARAM = "db"; + private static final String MODE_PARAM = "mode"; + + public Kraken2Step(PipelineStepProvider provider, PipelineContext ctx) + { + super(provider, ctx, new Kraken2Wrapper(ctx.getLogger())); + } + + public static class Provider extends AbstractPipelineStepProvider + { + public Provider() + { + super("Kraken2", "Kraken2", "Kraken2", "This step aligns input reads against a reference using BWA-mem and will only return read pairs without a passing hit in either read.", Arrays.asList( + ToolParameterDescriptor.create(DB_PARAM, "Database", "This determines the DB for positive or negative selection", "ldk-simplecombo", new JSONObject(){{ + put("storeValues", "bacteria-viral"); + put("multiSelect", false); + put("allowBlank", false); + put("joinReturnValue", true); + put("delimiter", ";"); + }}, "bacteria-viral"), + ToolParameterDescriptor.create(MODE_PARAM, "Reads To Retain", "This determines which set of reads is passed to the next step. If 'Retain Classified' is selected, then reads matching the DB are retained. if 'Retain Unclassified' is selected, then reads that do not match the DB are retained", "ldk-simplecombo", new JSONObject(){{ + put("storeValues", "Classified;Unclassified"); + put("multiSelect", false); + put("allowBlank", false); + put("joinReturnValue", true); + put("delimiter", ";"); + }}, null), + ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--minimum-hit-groups"), "minimumHitGroups", "Minimum Hit Groups", "Minimum number of hit groups (overlapping k-mers sharing the same minimizer) needed to make a call", "ldk-integerfield", new JSONObject(){{ + put("minValue", 0); + }}, 2), + ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--confidence"), "confidence", "Confidence", "Confidence score threshold (0-1)", "ldk-numberfield", new JSONObject(){{ + put("minValue", 0); + put("maxValue", 1); + put("decimalPrecision", 2); + }}, 0) + ), null, "https://github.com/DerrickWood/kraken2"); + } + + @Override + public Kraken2Step create(PipelineContext context) + { + return new Kraken2Step(this, context); + } + } + + @Override + public Output processInputFile(File inputFile, @Nullable File inputFile2, File outputDir) throws PipelineJobException + { + PreprocessingOutputImpl output = new PreprocessingOutputImpl(inputFile, inputFile2); + + List args = new ArrayList<>(); + args.add(getWrapper().getExe().getPath()); + + if (inputFile2 != null) + { + args.add("--paired"); + } + + if (inputFile.getName().toLowerCase().endsWith(".gz")) + { + args.add("--gzip-compressed"); + } + + Integer threads = SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger()); + if (threads != null) + { + args.add("--threads"); + args.add(threads.toString()); + } + + String dbName = getProvider().getParameterByName(DB_PARAM).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class); + if (dbName == null) + { + throw new PipelineJobException("Missing DB name"); + } + + File binDir = FileUtil.appendName(new File(PipelineJobService.get().getAppProperties().getToolsDirectory()), "kraken2_dbs"); + if (!binDir.exists()) + { + throw new PipelineJobException("Unable to find kraken2 DB dir, expected: " + binDir.getAbsolutePath()); + } + + File dbDir = FileUtil.appendName(binDir, dbName); + if (!dbDir.exists()) + { + throw new PipelineJobException("Unable to find kraken2 DB dir, expected: " + dbDir.getAbsolutePath()); + } + + args.add("--use-names"); + + args.add("--db"); + args.add(dbDir.getAbsolutePath()); + + args.addAll(getClientCommandArgs()); + + String mode = getProvider().getParameterByName(MODE_PARAM).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class); + + File unclassifiedOutputBase = FileUtil.appendName(outputDir, SequenceAnalysisService.get().getUnzippedBaseName(inputFile.getName()) + ".unclassified"); + args.add("--unclassified-out"); + args.add(unclassifiedOutputBase.getPath() + "#.fq.gz"); + + File classifiedOutputBase = FileUtil.appendName(outputDir, SequenceAnalysisService.get().getUnzippedBaseName(inputFile.getName()) + ".classified"); + args.add("--classified-out"); + args.add(classifiedOutputBase.getPath() + "#.fq.gz"); + + File reportFile = FileUtil.appendName(outputDir, SequencePipelineService.get().getUnzippedBaseName(inputFile.getName()) + ".kraken2.report.txt"); + args.add("--report"); + args.add(reportFile.getPath()); + + args.add(inputFile.getPath()); + if (inputFile2 != null) + { + args.add(inputFile2.getPath()); + } + + getWrapper().execute(args); + + File unclassified1 = new File(unclassifiedOutputBase.getPath() + "_1.fq.gz"); + File unclassified2 = inputFile2 == null ? null : new File(unclassifiedOutputBase.getPath() + "_2.fq.gz"); + + File classified1 = new File(classifiedOutputBase.getPath() + "_1.fq.gz"); + File classified2 = inputFile2 == null ? null : new File(classifiedOutputBase.getPath() + "_2.fq.gz"); + if ("Classified".equals(mode)) + { + if (!classified1.exists()) + { + throw new PipelineJobException("Classified file does not exist: " + classified1.getAbsolutePath()); + } + + output.setProcessedFastq(Pair.of(classified1, classified2)); + output.addIntermediateFile(unclassified1); + if (unclassified2 != null) + { + output.addIntermediateFile(unclassified2); + } + } + else + { + if (!unclassified1.exists()) + { + throw new PipelineJobException("Unclassified file does not exist: " + unclassified1.getAbsolutePath()); + } + + output.setProcessedFastq(Pair.of(unclassified1, unclassified2)); + output.addIntermediateFile(classified1); + if (classified2 != null) + { + output.addIntermediateFile(classified2); + } + } + + return output; + } + + public static class Kraken2Wrapper extends AbstractCommandWrapper + { + public Kraken2Wrapper(Logger log) + { + super(log); + } + + public File getExe() + { + return SimpleScriptWrapper.resolveFileInPath("kraken2", null, true); + } + } +} diff --git a/SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java b/SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java index 6c8f67114..a8c49a0c7 100644 --- a/SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java +++ b/SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java @@ -611,7 +611,7 @@ private void analysisPanelTest() throws Exception waitForElementToDisappear(Ext4Helper.Locators.window("Add Steps")); Map fieldsetMap = new HashMap<>(); - String[] setNames = {"Adapter Trimming (Trimmomatic)", "Average Quality Filter", "Crop Reads", "Downsample Reads", "Filter Reads Matching Reference", "Head Crop", "Quality Trimming (Adaptive)", "Quality Trimming (Sliding Window)", "Read Length Filter"}; + String[] setNames = {"Adapter Trimming (Trimmomatic)", "Average Quality Filter", "Crop Reads", "Downsample Reads", "Filter Reads Matching Reference", "Head Crop", "Quality Trimming (Adaptive)", "Quality Trimming (Sliding Window)", "Read Length Filter", "Kraken2"}; isPresentInThisOrder(setNames); for (String name : setNames) @@ -628,6 +628,9 @@ private void analysisPanelTest() throws Exception waitAndClick(Locator.id(fieldsetMap.get("Head Crop").down("ldk-linkbutton[text='Remove']", Ext4CmpRef.class).getId()).append(Locator.tag("a"))); waitForElementToDisappear(Locator.id(fieldsetMap.get("Head Crop").getId())); + waitAndClick(Locator.id(fieldsetMap.get("Kraken2").down("ldk-linkbutton[text='Remove']", Ext4CmpRef.class).getId()).append(Locator.tag("a"))); + waitForElementToDisappear(Locator.id(fieldsetMap.get("Kraken2").getId())); + Integer overlapLength = 6; Double errorRate = 0.2; Integer cropLength = 500; From f99dde4fad14f3f5d6c41096649cc40fbf3772fe Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 22 Jun 2026 07:47:33 -0700 Subject: [PATCH 12/18] actions/checkout@v3 -> v7 --- .github/workflows/branch_release.yml | 2 +- .github/workflows/merge_release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/branch_release.yml b/.github/workflows/branch_release.yml index e0b4cfc12..23e4616bf 100644 --- a/.github/workflows/branch_release.yml +++ b/.github/workflows/branch_release.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v7 - name: Create branches and PRs uses: LabKey/gitHubActions/branch-release@develop diff --git a/.github/workflows/merge_release.yml b/.github/workflows/merge_release.yml index f6d8d6ea0..2eee13940 100644 --- a/.github/workflows/merge_release.yml +++ b/.github/workflows/merge_release.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v7 - name: Merge PR uses: LabKey/gitHubActions/merge-release@develop From e3367f793f4cdf4261654439a2278b6baad58510 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 22 Jun 2026 09:17:08 -0700 Subject: [PATCH 13/18] Update node dependencies --- .github/workflows/build.yml | 8 +- jbrowse/package-lock.json | 135 ++++++++++++++-------------- jbrowse/package.json | 3 +- jbrowse/tools/buildStandaloneCli.js | 68 -------------- 4 files changed, 73 insertions(+), 141 deletions(-) delete mode 100644 jbrowse/tools/buildStandaloneCli.js diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7a01ac2f6..b50a98f37 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,9 +56,13 @@ jobs: LK_VERSION=$(cat /home/runner/work/_temp/_github_home/lkDist/release.txt | sed 's/-SNAPSHOT//g') cd /home/runner/work/_temp/_github_home/labkey_build/${LK_VERSION}/server/server/modules/DiscvrLabKeyModules COUNT=$(gh release list | grep 'latest' | wc -l) - if [ $COUNT != '0' ];then gh release delete 'latest' --cleanup-tag -y; fi + if [ $COUNT != '0' ];then + echo 'Deleting existing release' + gh release delete 'latest' --cleanup-tag -y; + fi - git push -f origin "latest" + git push -f origin 'latest' + sleep 100 # gh release periodically fails due to a missing tag. gh release create 'latest' --verify-tag --generate-notes --prerelease --title "Development Build: ${{ env.DEFAULT_BRANCH }}" gh release upload 'latest' /home/runner/work/_temp/_github_home/lkDist/discvr/DISCVR-* \ No newline at end of file diff --git a/jbrowse/package-lock.json b/jbrowse/package-lock.json index e0be57a60..6bbcb9fb9 100644 --- a/jbrowse/package-lock.json +++ b/jbrowse/package-lock.json @@ -18,13 +18,12 @@ "@jbrowse/product-core": "^4.1.13", "@jbrowse/react-linear-genome-view2": "^4.1.13", "@labkey/api": "^1.51.0", - "@labkey/components": "^7.35.0", + "@labkey/components": "^7.42.1", "@mui/x-data-grid": "^7.28.1", "assert": "^2.1.0", "browserify-zlib": "^0.2.0", "buffer": "^6.0.3", "child_process": "^1.0.2", - "fs": "^0.0.2", "jquery": "^3.7.1", "jspdf": "^4.2.1", "jspdf-autotable": "^5.0.7", @@ -84,21 +83,20 @@ } }, "node_modules/@babel/core": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", - "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "dev": true, - "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.29.0", - "@babel/generator": "^7.29.0", - "@babel/helper-compilation-targets": "^7.28.6", - "@babel/helper-module-transforms": "^7.28.6", - "@babel/helpers": "^7.28.6", - "@babel/parser": "^7.29.0", - "@babel/template": "^7.28.6", - "@babel/traverse": "^7.29.0", - "@babel/types": "^7.29.0", + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", @@ -375,14 +373,13 @@ } }, "node_modules/@babel/helpers": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.6.tgz", - "integrity": "sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", "dev": true, - "license": "MIT", "dependencies": { - "@babel/template": "^7.28.6", - "@babel/types": "^7.28.6" + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -3575,9 +3572,9 @@ } }, "node_modules/@labkey/api": { - "version": "1.51.3", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.51.3.tgz", - "integrity": "sha512-rs0idz4TLQuDatYBnLHFTL8ljpvCaCVynRA1KukTWD8yoZOGnMipX4m6+2CmoUaatZPFbiEIYvEqGU8WKLWrpg==" + "version": "1.51.4", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.51.4.tgz", + "integrity": "sha512-D8Rs1yd8dKnmLxGzMaTtOg3S7EpQRSGMCGPRNTPqICTukYwlHhi39UViiF9b7s3knlK4gtp9D64CHbQcehgw+w==" }, "node_modules/@labkey/build": { "version": "9.1.4", @@ -3616,12 +3613,12 @@ } }, "node_modules/@labkey/components": { - "version": "7.41.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.41.0.tgz", - "integrity": "sha512-lqYxEPIk0lyVFqA+mc+cJn+LUIuIQJrJNc3U0x3TDpKwt3OUbzsuhl51MrZwB21a1skM37HiYE5utysnQI48Ug==", + "version": "7.42.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.42.1.tgz", + "integrity": "sha512-y9sVFxdqCQDBnWCSW4BqtpauFMbjuTmywXDT1O6Zh31PopyyWhGiEA99K+zPxGzn7QRKh6BLcm0QrlTFPh0+jA==", "dependencies": { "@hello-pangea/dnd": "18.0.1", - "@labkey/api": "1.51.3", + "@labkey/api": "1.51.4", "@testing-library/dom": "~10.4.1", "@testing-library/jest-dom": "~6.9.1", "@testing-library/react": "~16.3.2", @@ -3631,7 +3628,7 @@ "date-fns": "~3.6.0", "date-fns-tz": "~3.2.0", "font-awesome": "~4.7.0", - "immer": "~10.1.3", + "immer": "~11.1.8", "immutable": "~3.8.3", "normalizr": "~3.6.2", "numeral": "~2.0.6", @@ -3640,11 +3637,11 @@ "react-color": "~2.19.3", "react-datepicker": "~7.6.0", "react-dom": "~18.3.1", - "react-router-dom": "~6.30.1", + "react-router-dom": "~6.30.4", "react-select": "~5.10.2", "react-treebeard": "~3.2.4", "vis-data": "~8.0.3", - "vis-network": "~10.0.2" + "vis-network": "~10.1.0" } }, "node_modules/@labkey/components/node_modules/immutable": { @@ -7139,9 +7136,9 @@ } }, "node_modules/dompurify": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.2.tgz", - "integrity": "sha512-lHeS9SA/IKeIFFyYciHBr2n0v1VMPlSj843HdLOwjb2OxNwdq9Xykxqhk+FE42MzAdHvInbAolSE4mhahPpjXA==", + "version": "3.4.11", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.11.tgz", + "integrity": "sha512-zhlUV12GsaRzMsf9q5M254YhA4+VuF0fG+QFqu6aYpoGlKtz+w8//jBcGVYBgQkR5GHjUomejY84AV+/uPbWdw==", "optionalDependencies": { "@types/trusted-types": "^2.0.7" } @@ -7865,11 +7862,6 @@ "node": ">= 0.6" } }, - "node_modules/fs": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.2.tgz", - "integrity": "sha512-YAiVokMCrSIFZiroB1oz51hPiPRVcUtSa4x2U5RYXyhS9VAPdiFigKbPTnOSq7XY8wd3FIVPYmXpo5lMzFmxgg==" - }, "node_modules/fs-extra": { "version": "11.3.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.1.tgz", @@ -8336,11 +8328,10 @@ } }, "node_modules/http-proxy-middleware": { - "version": "2.0.9", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz", - "integrity": "sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==", + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.10.tgz", + "integrity": "sha512-RKzRWNPxUZqbuk3BC5mGVJbBnWgr+diEnjJexIOytFbBzDy88Fbh/YvBr3DsNrl1jYAfjWfpATEv0NO35FDuPQ==", "dev": true, - "license": "MIT", "dependencies": { "@types/http-proxy": "^1.17.8", "http-proxy": "^1.18.1", @@ -8426,9 +8417,9 @@ "license": "BSD-3-Clause" }, "node_modules/immer": { - "version": "10.1.3", - "resolved": "https://registry.npmjs.org/immer/-/immer-10.1.3.tgz", - "integrity": "sha512-tmjF/k8QDKydUlm3mZU+tjM6zeq9/fFpPqH9SzWmBnVVKsPBg/V66qsMwb3/Bo90cgUN+ghdVBess+hPsxUyRw==", + "version": "11.1.8", + "resolved": "https://registry.npmjs.org/immer/-/immer-11.1.8.tgz", + "integrity": "sha512-/tbkHMW7y10Lx6i1crLjD4/OhNkRG+Fo7byZHtah0547nIeXYcpIXaUh0IAQY6gO5459qpGGYapcEOHtFXkIuA==", "funding": { "type": "opencollective", "url": "https://opencollective.com/immer" @@ -8839,10 +8830,20 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz", + "integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "dependencies": { "argparse": "^2.0.1" }, @@ -8981,14 +8982,13 @@ } }, "node_modules/launch-editor": { - "version": "2.12.0", - "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.12.0.tgz", - "integrity": "sha512-giOHXoOtifjdHqUamwKq6c49GzBdLjvxrd2D+Q4V6uOHopJv7p9VJxikDsQ/CBXZbEITgUqSVHXLTG3VhPP1Dg==", + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.14.1.tgz", + "integrity": "sha512-QWBrQsMpH7gPr965dsKD/3cKWiNoTjpATQf++Xq63N6sKRGMwlVXz41O1IZTMfZQgBctD/K5Zt06+/I6pP6+HA==", "dev": true, - "license": "MIT", "dependencies": { "picocolors": "^1.1.1", - "shell-quote": "^1.8.3" + "shell-quote": "^1.8.4" } }, "node_modules/librpc-web-mod": { @@ -11577,11 +11577,10 @@ } }, "node_modules/shell-quote": { - "version": "1.8.3", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz", - "integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==", + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.4.tgz", + "integrity": "sha512-VsC6n6vz1ihYYyZZwX7YZSF5l5x36ca17OC+a69h94YqB7X6XLwf+5MOgynYir2SLFUbl8gIYvBo8K8RoNQ6bQ==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -11969,11 +11968,10 @@ } }, "node_modules/tar": { - "version": "7.5.11", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", - "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", + "version": "7.5.16", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.16.tgz", + "integrity": "sha512-56adEpPMouktRlBLXiaYFFzZ/3+JXa8P9n7WbR+ibIjtviN55mEaOkiysCnPnWm+7kkui1Dn8J9l+g6zV8731w==", "dev": true, - "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", @@ -12570,9 +12568,9 @@ } }, "node_modules/vis-network": { - "version": "10.0.3", - "resolved": "https://registry.npmjs.org/vis-network/-/vis-network-10.0.3.tgz", - "integrity": "sha512-Sk8qoyY4oi8o5dh8bHsKTTtHi8wWZABbLpH0Ac95ULqVuBIFbMng95QJmEKP8kIQlkjbq3ChJrk1bPmig9Cxig==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/vis-network/-/vis-network-10.1.0.tgz", + "integrity": "sha512-D7b5p/C6SwWv1BlH9EDdtP0Tje/PJzSBWKef9qy2DyTC14QB7KBcnAZxIyW2m7mFYyfoeR+k5GF747zDcIhaKA==", "funding": { "type": "opencollective", "url": "https://opencollective.com/visjs" @@ -12870,11 +12868,10 @@ } }, "node_modules/webpack-dev-server": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-5.2.4.tgz", - "integrity": "sha512-GqDPGZN9bRqKBTkp4aWkobDDHMsrXKoGSdOH56smIri8qR0JG8gfL8/v/f/OZR3/OKXjG8uwJbFVhKm/FNU/UA==", + "version": "5.2.5", + "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-5.2.5.tgz", + "integrity": "sha512-4wZtCquSuv9CKX8oybo+mqxtxZqWz47uM1Ch94lxowBztOhWCbhqvRbfC/mODOwxgV2brY+JGZpHq58/SuVFYg==", "dev": true, - "license": "MIT", "dependencies": { "@types/bonjour": "^3.5.13", "@types/connect-history-api-fallback": "^1.5.4", diff --git a/jbrowse/package.json b/jbrowse/package.json index 31c0c43e2..bd5a2a5e6 100644 --- a/jbrowse/package.json +++ b/jbrowse/package.json @@ -24,13 +24,12 @@ "@jbrowse/product-core": "^4.1.13", "@jbrowse/react-linear-genome-view2": "^4.1.13", "@labkey/api": "^1.51.0", - "@labkey/components": "^7.35.0", + "@labkey/components": "^7.42.1", "@mui/x-data-grid": "^7.28.1", "assert": "^2.1.0", "browserify-zlib": "^0.2.0", "buffer": "^6.0.3", "child_process": "^1.0.2", - "fs": "^0.0.2", "jquery": "^3.7.1", "jspdf": "^4.2.1", "jspdf-autotable": "^5.0.7", diff --git a/jbrowse/tools/buildStandaloneCli.js b/jbrowse/tools/buildStandaloneCli.js deleted file mode 100644 index fbfd342e9..000000000 --- a/jbrowse/tools/buildStandaloneCli.js +++ /dev/null @@ -1,68 +0,0 @@ -// This script is a workaround until pkg supports ES modules: https://github.com/vercel/pkg/issues/1291 -// See also: https://nodejs.org/api/single-executable-applications.html -// This should be replaced once a better solution is available -const fs = require('fs'); -const { exec, execSync } = require('child_process'); - -function convertEs6ToCommonJS(packageName) { - const sourceFile = './buildCli/node_modules/@isaacs/cliui/node_modules/' + packageName + '/index.js'; - const origFile = sourceFile + '.orig'; - if (fs.existsSync(origFile)) { - console.log('File exists, skipping: ' + origFile); - } else { - console.log('Updating: ' + sourceFile); - if (!fs.existsSync(sourceFile)) { - throw new Error('Missing file: ' + sourceFile); - } - - if (fs.existsSync('./buildCli/' + packageName)) { - console.log('Deleting existing index.js file') - fs.unlinkSync('./buildCli/' + packageName) - } - - try { - execSync('npx esbuild --outdir=./buildCli/' + packageName + ' --target=es6 --format=cjs ' + sourceFile) - } catch (err) { - throw new Error('ERROR: ' + err.message); - } - - try { - fs.renameSync(sourceFile, origFile) - } catch (err){ - throw new Error('ERROR: ' + err); - } - - try { - fs.renameSync('./buildCli/' + packageName + '/index.js', sourceFile) - } catch (err){ - throw new Error('ERROR: ' + err); - } - } -} - -convertEs6ToCommonJS('ansi-regex') -convertEs6ToCommonJS('ansi-styles') -convertEs6ToCommonJS('strip-ansi') -convertEs6ToCommonJS('wrap-ansi') -convertEs6ToCommonJS('string-width') - -const child = exec('npx pkg --outdir=./resources/external/jb-cli ./buildCli/node_modules/@jbrowse/cli', (err, stdout, stderr) => { - if (err) { - throw new Error('Unable to run pkg: ' + err) - } -}); - -child.stdout.setEncoding('utf8'); -child.stdout.on('data', function(data) { - console.log(data); -}); - -child.stderr.setEncoding('utf8'); -child.stderr.on('data', function(data) { - console.log(data); -}); - -child.on('close', function(code) { - console.log('closing pkg: ' + code); -}); - From f3c8acc13dc2beff27be2ab5ae181d31654f8be7 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 24 Jun 2026 10:31:46 -0700 Subject: [PATCH 14/18] Bugfixes to kraken2 --- .../run/preprocessing/Kraken2Step.java | 54 ++++++++++++------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java index 6f22f595f..b4732714d 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java @@ -16,6 +16,7 @@ import org.labkey.api.sequenceanalysis.run.AbstractCommandPipelineStep; import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; +import org.labkey.api.util.Compress; import org.labkey.api.util.FileUtil; import org.labkey.api.util.Pair; @@ -121,15 +122,23 @@ public Output processInputFile(File inputFile, @Nullable File inputFile2, File o args.addAll(getClientCommandArgs()); - String mode = getProvider().getParameterByName(MODE_PARAM).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class); + args.add("--output"); + args.add("-"); - File unclassifiedOutputBase = FileUtil.appendName(outputDir, SequenceAnalysisService.get().getUnzippedBaseName(inputFile.getName()) + ".unclassified"); - args.add("--unclassified-out"); - args.add(unclassifiedOutputBase.getPath() + "#.fq.gz"); + String mode = getProvider().getParameterByName(MODE_PARAM).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class); File classifiedOutputBase = FileUtil.appendName(outputDir, SequenceAnalysisService.get().getUnzippedBaseName(inputFile.getName()) + ".classified"); - args.add("--classified-out"); - args.add(classifiedOutputBase.getPath() + "#.fq.gz"); + File unclassifiedOutputBase = FileUtil.appendName(outputDir, SequenceAnalysisService.get().getUnzippedBaseName(inputFile.getName()) + ".unclassified"); + if ("Classified".equals(mode)) + { + args.add("--classified-out"); + args.add(classifiedOutputBase.getPath() + "#.fq"); + } + else + { + args.add("--unclassified-out"); + args.add(unclassifiedOutputBase.getPath() + "#.fq"); + } File reportFile = FileUtil.appendName(outputDir, SequencePipelineService.get().getUnzippedBaseName(inputFile.getName()) + ".kraken2.report.txt"); args.add("--report"); @@ -143,38 +152,45 @@ public Output processInputFile(File inputFile, @Nullable File inputFile2, File o getWrapper().execute(args); - File unclassified1 = new File(unclassifiedOutputBase.getPath() + "_1.fq.gz"); - File unclassified2 = inputFile2 == null ? null : new File(unclassifiedOutputBase.getPath() + "_2.fq.gz"); - - File classified1 = new File(classifiedOutputBase.getPath() + "_1.fq.gz"); - File classified2 = inputFile2 == null ? null : new File(classifiedOutputBase.getPath() + "_2.fq.gz"); if ("Classified".equals(mode)) { + File classified1 = new File(classifiedOutputBase.getPath() + "_1.fq"); + File classified2 = inputFile2 == null ? null : new File(classifiedOutputBase.getPath() + "_2.fq"); if (!classified1.exists()) { throw new PipelineJobException("Classified file does not exist: " + classified1.getAbsolutePath()); } - output.setProcessedFastq(Pair.of(classified1, classified2)); - output.addIntermediateFile(unclassified1); - if (unclassified2 != null) + File compressed1 = Compress.compressGzip(classified1); + output.addIntermediateFile(classified1); + + File compressed2 = classified2 == null ? null : Compress.compressGzip(classified2); + if (classified2 != null) { - output.addIntermediateFile(unclassified2); + output.addIntermediateFile(classified2); } + + output.setProcessedFastq(Pair.of(compressed1, compressed2)); } else { + File unclassified1 = new File(unclassifiedOutputBase.getPath() + "_1.fq"); + File unclassified2 = inputFile2 == null ? null : new File(unclassifiedOutputBase.getPath() + "_2.fq"); if (!unclassified1.exists()) { throw new PipelineJobException("Unclassified file does not exist: " + unclassified1.getAbsolutePath()); } - output.setProcessedFastq(Pair.of(unclassified1, unclassified2)); - output.addIntermediateFile(classified1); - if (classified2 != null) + File compressed1 = Compress.compressGzip(unclassified1); + output.addIntermediateFile(unclassified1); + + File compressed2 = unclassified2 == null ? null : Compress.compressGzip(unclassified2); + if (unclassified2 != null) { - output.addIntermediateFile(classified2); + output.addIntermediateFile(unclassified2); } + + output.setProcessedFastq(Pair.of(compressed1, compressed2)); } return output; From 84c92513c1c05f8b9f1c57806eaa80e68fc22246 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 24 Jun 2026 10:37:38 -0700 Subject: [PATCH 15/18] Adjust DB names for kraken2 --- .../sequenceanalysis/run/preprocessing/Kraken2Step.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java index b4732714d..dc57453d1 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java @@ -41,12 +41,12 @@ public Provider() { super("Kraken2", "Kraken2", "Kraken2", "This step aligns input reads against a reference using BWA-mem and will only return read pairs without a passing hit in either read.", Arrays.asList( ToolParameterDescriptor.create(DB_PARAM, "Database", "This determines the DB for positive or negative selection", "ldk-simplecombo", new JSONObject(){{ - put("storeValues", "bacteria-viral"); + put("storeValues", "kraken2_bv;kraken2_standard"); put("multiSelect", false); put("allowBlank", false); put("joinReturnValue", true); put("delimiter", ";"); - }}, "bacteria-viral"), + }}, "kraken2_bv"), ToolParameterDescriptor.create(MODE_PARAM, "Reads To Retain", "This determines which set of reads is passed to the next step. If 'Retain Classified' is selected, then reads matching the DB are retained. if 'Retain Unclassified' is selected, then reads that do not match the DB are retained", "ldk-simplecombo", new JSONObject(){{ put("storeValues", "Classified;Unclassified"); put("multiSelect", false); From a9aec60d67b2f9ba194569f88aae61bbb531a9b5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 24 Jun 2026 14:02:18 -0700 Subject: [PATCH 16/18] Add --memory-mapping for kraken2 --- .../labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java | 1 + 1 file changed, 1 insertion(+) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java index dc57453d1..5842b4228 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/Kraken2Step.java @@ -54,6 +54,7 @@ public Provider() put("joinReturnValue", true); put("delimiter", ";"); }}, null), + ToolParameterDescriptor.createCommandLineParam(CommandLineParam.createSwitch("--memory-mapping"), "memoryMapping", "Memory Mapping", "If checked, the DB will not be read into memory, reducing RAM", "checkbox", null, false), ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--minimum-hit-groups"), "minimumHitGroups", "Minimum Hit Groups", "Minimum number of hit groups (overlapping k-mers sharing the same minimizer) needed to make a call", "ldk-integerfield", new JSONObject(){{ put("minValue", 0); }}, 2), From dc788ef8a899c983208b3246b14d2932aa81e8fd Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 25 Jun 2026 14:02:32 -0700 Subject: [PATCH 17/18] Improve handling of 10x chemistry --- .../resources/queries/singlecell/cdna_libraries/.qview.xml | 1 + .../queries/singlecell/cdna_libraries/Cohort Info.qview.xml | 1 + singlecell/resources/web/singlecell/panel/PoolImportPanel.js | 5 ++--- singlecell/resources/web/singlecell/panel/cDNAImportPanel.js | 4 ++++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/singlecell/resources/queries/singlecell/cdna_libraries/.qview.xml b/singlecell/resources/queries/singlecell/cdna_libraries/.qview.xml index 3226dbcf0..93d777b34 100644 --- a/singlecell/resources/queries/singlecell/cdna_libraries/.qview.xml +++ b/singlecell/resources/queries/singlecell/cdna_libraries/.qview.xml @@ -14,6 +14,7 @@ + diff --git a/singlecell/resources/queries/singlecell/cdna_libraries/Cohort Info.qview.xml b/singlecell/resources/queries/singlecell/cdna_libraries/Cohort Info.qview.xml index 692e8d8e6..eecc93482 100644 --- a/singlecell/resources/queries/singlecell/cdna_libraries/Cohort Info.qview.xml +++ b/singlecell/resources/queries/singlecell/cdna_libraries/Cohort Info.qview.xml @@ -16,6 +16,7 @@ + diff --git a/singlecell/resources/web/singlecell/panel/PoolImportPanel.js b/singlecell/resources/web/singlecell/panel/PoolImportPanel.js index 5ee0ccfd5..8fd230b87 100644 --- a/singlecell/resources/web/singlecell/panel/PoolImportPanel.js +++ b/singlecell/resources/web/singlecell/panel/PoolImportPanel.js @@ -149,8 +149,7 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { allowRowSpan: true },{ name: 'kitType', - labels: ['Kit Type', 'V1.1/V2/HT', 'V1.1/HT', 'HT/V1.1/V2', 'V2/HT'], - transform: 'kitType' + labels: ['Kit Type', 'V1.1/V2/HT', 'V1.1/HT', 'HT/V1.1/V2', 'V2/HT'] }], IGNORED_COLUMNS: [], @@ -1045,7 +1044,7 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { LDK.Assert.assertNotEmpty('Expected non-null workbook', row.workbook); var cDNA = Ext4.apply({ sortGUID: sortMap[sortKey], - chemistry: null, + chemistry: row.kitType, plateId: row.plateId, well: row.well || 'Pool', citeseqpanel: row.citeseqpanel, diff --git a/singlecell/resources/web/singlecell/panel/cDNAImportPanel.js b/singlecell/resources/web/singlecell/panel/cDNAImportPanel.js index a02351135..a76717c2d 100644 --- a/singlecell/resources/web/singlecell/panel/cDNAImportPanel.js +++ b/singlecell/resources/web/singlecell/panel/cDNAImportPanel.js @@ -403,6 +403,10 @@ Ext4.define('SingleCell.panel.cDNAImportPanel', { citeseqReadsetId: row.citeseqReadsetId }; + if (row.kitType) { + baseRow.chemistry = row.kitType + } + var gexReadsetId = readsetMap[row.plateId + '-GEX']; if (gexReadsetId) { baseRow.readsetId = gexReadsetId; From e7570d05a3578a440fdbe1a22ad94e33d72b867c Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 3 Jul 2026 08:43:14 -0700 Subject: [PATCH 18/18] Check whether study exists in study overview --- Studies/resources/views/studiesOverview.html | 58 +++++++++++--------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/Studies/resources/views/studiesOverview.html b/Studies/resources/views/studiesOverview.html index 5fd144f1e..b2cb492ee 100644 --- a/Studies/resources/views/studiesOverview.html +++ b/Studies/resources/views/studiesOverview.html @@ -26,37 +26,43 @@ // } // }) - LABKEY.Query.selectRows({ - schemaName: 'study', - queryName: 'datasets', - columns: 'Name,Label,CategoryId/Label', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - let div = $('
') + // timepointType is an indirect test for whether a study exists: + if (LABKEY.moduleContext.study.timepointType) { + LABKEY.Query.selectRows({ + schemaName: 'study', + queryName: 'datasets', + columns: 'Name,Label,CategoryId/Label', + scope: this, + failure: LDK.Utils.getErrorCallback(), + success: function (results) { + let div = $('
') - const dataByCategory = {} - results.rows.sort((a, b) => a.Label.localeCompare(b.Label)).forEach(row => { - const dsUrl = LABKEY.ActionURL.buildURL('query', 'executeQuery', null, { - schemaName: 'study', - queryName: row.Name - }); + const dataByCategory = {} + results.rows.sort((a, b) => a.Label.localeCompare(b.Label)).forEach(row => { + const dsUrl = LABKEY.ActionURL.buildURL('query', 'executeQuery', null, { + schemaName: 'study', + queryName: row.Name + }); - dataByCategory[row['CategoryId/Label']] = dataByCategory[row['CategoryId/Label']] || [] - dataByCategory[row['CategoryId/Label']].push($(`
  • ${row.Label}
  • `)) - }) + dataByCategory[row['CategoryId/Label']] = dataByCategory[row['CategoryId/Label']] || [] + dataByCategory[row['CategoryId/Label']].push($(`
  • ${row.Label}
  • `)) + }) - Object.keys(dataByCategory).sort().forEach(category => { - div.append('

    ' + category + '

    ').append('
      ') - const ul = div.children('ul')[0] - dataByCategory[category].forEach(item => { - ul.append(item[0]) + Object.keys(dataByCategory).sort().forEach(category => { + div.append('

      ' + category + '

      ').append('
        ') + const ul = div.children('ul')[0] + dataByCategory[category].forEach(item => { + ul.append(item[0]) + }) }) - }) - $('#' + webpart.wrapperDivId).append(div) - } - }) + $('#' + webpart.wrapperDivId).append(div) + } + }) + } else { + let div = $('
        There are no studies in the current folder
        ') + $('#' + webpart.wrapperDivId).append(div) + } }) }); }(jQuery, LABKEY));