Skip to content

Commit 6953cd9

Browse files
ErinWeisbartErinWeisbart
committed
support additional grouping level for analysis jobs
1 parent 17388b3 commit 6953cd9

1 file changed

Lines changed: 108 additions & 32 deletions

File tree

‎run_batch_general.py‎

Lines changed: 108 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ def run_batch_general(
3737
columns=range(1, 25), # (Not used by illum.)
3838
wells="", # (explicitly list wells. Overwrites rows and columns if passed. Not used by illum. e.g. ['B3','C7'])
3939
sites=range(1, 10), # (Not used by illum, qc, or assaydev.)
40+
extra_grouping=False,
41+
grouping=range(1, 5), # (Not used by illum, qc, or assaydev.)
4042
well_digit_pad=True, # Set True to A01 well format name, set False to A1
4143
pipeline="", # (overwrite default pipeline names)
4244
pipelinepath="", # (overwrite default path to pipelines)
@@ -494,10 +496,16 @@ def run_batch_general(
494496

495497
elif step == "analysis":
496498
analysisqueue = JobQueue(f"{identifier}_Analysis")
497-
if not outputstructure:
498-
outputstructure = (
499-
"Metadata_Plate/analysis/Metadata_Plate-Metadata_Well-Metadata_Site"
500-
)
499+
if not extra_grouping:
500+
if not outputstructure:
501+
outputstructure = (
502+
"Metadata_Plate/analysis/Metadata_Plate-Metadata_Well-Metadata_Site"
503+
)
504+
else:
505+
if not outputstructure:
506+
outputstructure = (
507+
"Metadata_Plate/analysis/Metadata_Plate-Metadata_Well-Metadata_Site-Metadata_Group"
508+
)
501509
if not outpath:
502510
outpath = path_dict[path_style]["analysisoutpath"]
503511
if not usebatch:
@@ -510,8 +518,37 @@ def run_batch_general(
510518
for eachrow in rows:
511519
for eachcol in columns:
512520
for eachsite in sites:
521+
if not extra_grouping:
522+
templateMessage_analysis = {
523+
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}},Metadata_Site={str(eachsite)}",
524+
"pipeline": posixpath.join(pipelinepath, pipeline),
525+
"output": outpath,
526+
"output_structure": outputstructure,
527+
"input": inputpath,
528+
"data_file": posixpath.join(
529+
datafilepath, plate, csvname
530+
),
531+
}
532+
analysisqueue.scheduleBatch(templateMessage_analysis)
533+
else:
534+
for eachgroup in grouping:
535+
templateMessage_analysis = {
536+
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}},Metadata_Site={str(eachsite)},Metadata_Group={str(eachgroup)}",
537+
"pipeline": posixpath.join(pipelinepath, pipeline),
538+
"output": outpath,
539+
"output_structure": outputstructure,
540+
"input": inputpath,
541+
"data_file": posixpath.join(
542+
datafilepath, plate, csvname
543+
),
544+
}
545+
analysisqueue.scheduleBatch(templateMessage_analysis)
546+
else:
547+
for eachwell in wells:
548+
for eachsite in sites:
549+
if not extra_grouping:
513550
templateMessage_analysis = {
514-
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}},Metadata_Site={str(eachsite)}",
551+
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell},Metadata_Site={str(eachsite)}",
515552
"pipeline": posixpath.join(pipelinepath, pipeline),
516553
"output": outpath,
517554
"output_structure": outputstructure,
@@ -521,20 +558,19 @@ def run_batch_general(
521558
),
522559
}
523560
analysisqueue.scheduleBatch(templateMessage_analysis)
524-
else:
525-
for eachwell in wells:
526-
for eachsite in sites:
527-
templateMessage_analysis = {
528-
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell},Metadata_Site={str(eachsite)}",
529-
"pipeline": posixpath.join(pipelinepath, pipeline),
530-
"output": outpath,
531-
"output_structure": outputstructure,
532-
"input": inputpath,
533-
"data_file": posixpath.join(
534-
datafilepath, plate, csvname
535-
),
536-
}
537-
analysisqueue.scheduleBatch(templateMessage_analysis)
561+
else:
562+
for eachgroup in grouping:
563+
templateMessage_analysis = {
564+
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell},Metadata_Site={str(eachsite)},Metadata_Group={str(eachgroup)}",
565+
"pipeline": posixpath.join(pipelinepath, pipeline),
566+
"output": outpath,
567+
"output_structure": outputstructure,
568+
"input": inputpath,
569+
"data_file": posixpath.join(
570+
datafilepath, plate, csvname
571+
),
572+
}
573+
analysisqueue.scheduleBatch(templateMessage_analysis)
538574
else:
539575
if not batchfile:
540576
batchfile = "Batch_data_analysis.h5"
@@ -543,27 +579,51 @@ def run_batch_general(
543579
for eachrow in rows:
544580
for eachcol in columns:
545581
for eachsite in sites:
582+
if not extra_grouping:
583+
templateMessage_analysis = {
584+
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}},Metadata_Site={str(eachsite)}",
585+
"pipeline": posixpath.join(batchpath, batchfile),
586+
"output": outpath,
587+
"output_structure": outputstructure,
588+
"input": inputpath,
589+
"data_file": posixpath.join(batchpath, batchfile),
590+
}
591+
analysisqueue.scheduleBatch(templateMessage_analysis)
592+
else:
593+
for eachgroup in grouping:
594+
templateMessage_analysis = {
595+
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}},Metadata_Site={str(eachsite)},Metadata_Group={str(eachgroup)}",
596+
"pipeline": posixpath.join(batchpath, batchfile),
597+
"output": outpath,
598+
"output_structure": outputstructure,
599+
"input": inputpath,
600+
"data_file": posixpath.join(batchpath, batchfile),
601+
}
602+
analysisqueue.scheduleBatch(templateMessage_analysis)
603+
else:
604+
for eachwell in wells:
605+
for eachsite in sites:
606+
if not extra_grouping:
546607
templateMessage_analysis = {
547-
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}},Metadata_Site={str(eachsite)}",
608+
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell},Metadata_Site={str(eachsite)}",
548609
"pipeline": posixpath.join(batchpath, batchfile),
549610
"output": outpath,
550611
"output_structure": outputstructure,
551612
"input": inputpath,
552613
"data_file": posixpath.join(batchpath, batchfile),
553614
}
554615
analysisqueue.scheduleBatch(templateMessage_analysis)
555-
else:
556-
for eachwell in wells:
557-
for eachsite in sites:
558-
templateMessage_analysis = {
559-
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell},Metadata_Site={str(eachsite)}",
560-
"pipeline": posixpath.join(batchpath, batchfile),
561-
"output": outpath,
562-
"output_structure": outputstructure,
563-
"input": inputpath,
564-
"data_file": posixpath.join(batchpath, batchfile),
565-
}
566-
analysisqueue.scheduleBatch(templateMessage_analysis)
616+
else:
617+
for eachgroup in grouping:
618+
templateMessage_analysis = {
619+
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell},Metadata_Site={str(eachsite)},Metadata_Group={str(eachgroup)}",
620+
"pipeline": posixpath.join(batchpath, batchfile),
621+
"output": outpath,
622+
"output_structure": outputstructure,
623+
"input": inputpath,
624+
"data_file": posixpath.join(batchpath, batchfile),
625+
}
626+
analysisqueue.scheduleBatch(templateMessage_analysis)
567627

568628
print("Analysis job submitted. Check your queue")
569629

@@ -628,6 +688,20 @@ def run_batch_general(
628688
default="1,2,3,4,5,6,7,8,9",
629689
help="List of site to process",
630690
)
691+
parser.add_argument(
692+
"--extra-grouping",
693+
dest="extra_grouping",
694+
action="store_true",
695+
default=False,
696+
help="Add extra grouping to the job queue. Supported for analysis only.",
697+
)
698+
parser.add_argument(
699+
"--grouping",
700+
dest="grouping",
701+
type=lambda s: list(s.split(",")),
702+
default="1,2,3,4",
703+
help="List of groups to process",
704+
)
631705
parser.add_argument(
632706
"--no-well-digit-pad",
633707
dest="well_digit_pad",
@@ -710,6 +784,8 @@ def run_batch_general(
710784
columns=args.columns,
711785
wells=args.wells,
712786
sites=args.sites,
787+
extra_grouping=args.extra_grouping,
788+
grouping=args.grouping,
713789
well_digit_pad=args.well_digit_pad,
714790
pipeline=args.pipeline,
715791
pipelinepath=args.pipelinepath,

0 commit comments

Comments
 (0)