Skip to content

add run_batch script formatted for CPG #179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ErinWeisbart merged 19 commits into from
Nov 5, 2024
Merged

add run_batch script formatted for CPG #179

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
6416ad7
add run_batch script formatted for CPG
ErinWeisbart Aug 22, 2024
ccb124f
add run_batch_CPG to docs
ErinWeisbart Aug 22, 2024
a3ae1f7
run_batch_general to CLI, cleaned up
ErinWeisbart Aug 27, 2024
b695c03
black
ErinWeisbart Aug 27, 2024
d84b695
run_batch_general to CLI docs
ErinWeisbart Aug 27, 2024
dd15e19
typo
ErinWeisbart Aug 30, 2024
0575cb1
typo
ErinWeisbart Aug 30, 2024
cf11b2b
list
ErinWeisbart Aug 30, 2024
4c76aaa
typo
ErinWeisbart Oct 8, 2024
1cb103c
argument consistency
ErinWeisbart Oct 21, 2024
74876ee
more logical ordering
ErinWeisbart Oct 21, 2024
196050a
default empty
ErinWeisbart Oct 21, 2024
fe03c0f
make docs match argument cleanup
ErinWeisbart Oct 21, 2024
b47fbfe
make ints for checking
ErinWeisbart Oct 21, 2024
dabd821
cleanup from review
ErinWeisbart Oct 22, 2024
ceb6a42
add explicit well option
ErinWeisbart Oct 22, 2024
bd8d89c
add error catch
ErinWeisbart Oct 23, 2024
5d4c2b0
properly check no wells as ['']
ErinWeisbart Oct 24, 2024
afd3755
default default
ErinWeisbart Oct 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add explicit well option
  • Loading branch information
@ErinWeisbart
ErinWeisbart committed Oct 22, 2024
commit ceb6a428d0b3b8f91c7abd181f883a8a84113939
7 changes: 6 additions & 1 deletion documentation/DCP-documentation/step_2_submit_jobs.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,12 @@ Separate values with commas and no spaces and surround with quotation marks (e.g
* `--columns <value>`: a custom list of column labels.
Will be combined with `rows` to generate well names.
Separate values with commas and no spaces and surround with quotation marks (e.g. `"1,2,3,4,5,6,7,8,9,10"`)
* `--no-well-digit-pad`: Formats wells without well digit padding (e.g. `A1` NOT `A01`)
* `--wells <value>`: a custom list of wells.
Overwrites `rows` and `columns`.
Separate values with commas and no spaces and surround with quotation marks (e.g. `"C02,D04,E04,N12"`)
* `--no-well-digit-pad`: Formats wells without well digit padding.
Formats wells passed with `--plate format` or `--rows` and `--columns` but not `--wells`.
(e.g. `A1` NOT `A01`)
* `--sites <value>`: a custom list of sites (fields of view) to be analyzed.
Separate values with commas and no spaces and surround with quotation marks (e.g. `"1,2,3,4,5,6"`)

Expand Down
205 changes: 170 additions & 35 deletions run_batch_general.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,11 @@ def run_batch_general(
batch="", # (e.g. 2020_11_04_CPJUMP1)
platelist=[], # (e.g. ['Plate1','Plate2'])
source="", # (e.g. source_4, broad. Only with path_style=="cpg")
plate_format="", # (96 or 384. Overwrites rows and columns if passed.)
rows=list(string.ascii_uppercase)[0:16],
columns=range(1, 25),
sites=range(1, 10),
plate_format="", # (96 or 384. Overwrites rows and columns if passed. Not used by illum.)
rows=list(string.ascii_uppercase)[0:16], # (Not used by illum.)
columns=range(1, 25), # (Not used by illum.)
wells="", # (explicitly list wells. Overwrites rows and columns if passed. Not used by illum. e.g. ['B3','C7'])
sites=range(1, 10), # (Not used by illum, qc, or assaydev.)
well_digit_pad=True, # Set True to A01 well format name, set False to A1
pipeline="", # (overwrite default pipeline names)
pipelinepath="", # (overwrite default path to pipelines)
Expand Down Expand Up @@ -137,11 +138,26 @@ def run_batch_general(
csvname = "load_data_unprojected.csv"

for plate in platelist:
for eachrow in rows:
for eachcol in columns:
if not wells:
for eachrow in rows:
for eachcol in columns:
for eachsite in sites:
templateMessage_zproj = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}},Metadata_Site={str(eachsite)}",
"pipeline": posixpath.join(pipelinepath, pipeline),
"output": outpath,
"output_structure": outputstructure,
"input": inputpath,
"data_file": posixpath.join(
datafilepath, plate, csvname
),
}
zprojqueue.scheduleBatch(templateMessage_zproj)
else:
for eachwell in wells:
for eachsite in sites:
templateMessage_zproj = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{eachcol:{well_format}},Metadata_Site={str(eachsite)}",
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell},Metadata_Site={str(eachsite)}",
"pipeline": posixpath.join(pipelinepath, pipeline),
"output": outpath,
"output_structure": outputstructure,
Expand All @@ -150,16 +166,29 @@ def run_batch_general(
datafilepath, plate, csvname
),
}
zprojqueue.scheduleBatch(templateMessage_zproj)
zprojqueue.scheduleBatch(templateMessage_zproj)
else:
if not batchfile:
batchfile = "Batch_data_zproj.h5"
for plate in platelist:
for eachrow in rows:
for eachcol in columns:
if not wells:
for eachrow in rows:
for eachcol in columns:
for eachsite in sites:
templateMessage_zproj = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}},Metadata_Site={str(eachsite)}",
"pipeline": posixpath.join(batchpath, batchfile),
"output": outpath,
"output_structure": outputstructure,
"input": inputpath,
"data_file": posixpath.join(batchpath, batchfile),
}
zprojqueue.scheduleBatch(templateMessage_zproj)
else:
for eachwell in wells:
for eachsite in sites:
templateMessage_zproj = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{eachcol:{well_format}},Metadata_Site={str(eachsite)}",
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell},Metadata_Site={str(eachsite)}",
"pipeline": posixpath.join(batchpath, batchfile),
"output": outpath,
"output_structure": outputstructure,
Expand Down Expand Up @@ -215,10 +244,21 @@ def run_batch_general(
csvname = "load_data.csv"

for plate in platelist:
for eachrow in rows:
for eachcol in columns:
if not wells:
for eachrow in rows:
for eachcol in columns:
templateMessage_qc = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}}",
"pipeline": posixpath.join(pipelinepath, pipeline),
"output": outpath,
"input": inputpath,
"data_file": posixpath.join(datafilepath, plate, csvname),
}
qcqueue.scheduleBatch(templateMessage_qc)
else:
for eachwell in wells:
templateMessage_qc = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{eachcol:{well_format}}",
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell}",
"pipeline": posixpath.join(pipelinepath, pipeline),
"output": outpath,
"input": inputpath,
Expand All @@ -229,10 +269,21 @@ def run_batch_general(
if not batchfile:
batchfile = "Batch_data_qc.h5"
for plate in platelist:
for eachrow in rows:
for eachcol in columns:
if not wells:
for eachrow in rows:
for eachcol in columns:
templateMessage_qc = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}}",
"pipeline": posixpath.join(batchpath, batchfile),
"output": outpath,
"input": inputpath,
"data_file": posixpath.join(batchpath, batchfile),
}
qcqueue.scheduleBatch(templateMessage_qc)
else:
for well in wells:
templateMessage_qc = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{eachcol:{well_format}}",
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell}",
"pipeline": posixpath.join(batchpath, batchfile),
"output": outpath,
"input": inputpath,
Expand All @@ -253,11 +304,25 @@ def run_batch_general(
csvname = "load_data.csv"

for plate in platelist:
for eachrow in rows:
for eachcol in columns:
if not wells:
for eachrow in rows:
for eachcol in columns:
for eachsite in sites:
templateMessage_qc = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}},Metadata_Site={str(eachsite)}",
"pipeline": posixpath.join(pipelinepath, pipeline),
"output": outpath,
"input": inputpath,
"data_file": posixpath.join(
datafilepath, plate, csvname
),
}
qcqueue.scheduleBatch(templateMessage_qc)
else:
for well in wells:
for eachsite in sites:
templateMessage_qc = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{eachcol:{well_format}},Metadata_Site={str(eachsite)}",
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell},Metadata_Site={str(eachsite)}",
"pipeline": posixpath.join(pipelinepath, pipeline),
"output": outpath,
"input": inputpath,
Expand All @@ -270,11 +335,23 @@ def run_batch_general(
if not batchfile:
batchfile = "Batch_data_qc.h5"
for plate in platelist:
for eachrow in rows:
for eachcol in columns:
if not wells:
for eachrow in rows:
for eachcol in columns:
for eachsite in sites:
templateMessage_qc = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}},Metadata_Site={str(eachsite)}",
"pipeline": posixpath.join(batchpath, batchfile),
"output": outpath,
"input": inputpath,
"data_file": posixpath.join(batchpath, batchfile),
}
qcqueue.scheduleBatch(templateMessage_qc)
else:
for well in wells:
for eachsite in sites:
templateMessage_qc = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{eachcol:{well_format}},Metadata_Site={str(eachsite)}",
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell},Metadata_Site={str(eachsite)}",
"pipeline": posixpath.join(batchpath, batchfile),
"output": outpath,
"input": inputpath,
Expand All @@ -295,10 +372,21 @@ def run_batch_general(
csvname = "load_data_with_illum.csv"

for plate in platelist:
for eachrow in rows:
for eachcol in columns:
if not wells:
for eachrow in rows:
for eachcol in columns:
templateMessage_ad = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}}",
"pipeline": posixpath.join(pipelinepath, pipeline),
"output": outpath,
"input": inputpath,
"data_file": posixpath.join(datafilepath, plate, csvname),
}
assaydevqueue.scheduleBatch(templateMessage_ad)
else:
for well in wells:
templateMessage_ad = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{eachcol:{well_format}}",
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell}",
"pipeline": posixpath.join(pipelinepath, pipeline),
"output": outpath,
"input": inputpath,
Expand All @@ -309,10 +397,21 @@ def run_batch_general(
if not batchfile:
batchfile = "Batch_data_assaydev.h5"
for plate in platelist:
for eachrow in rows:
for eachcol in columns:
if not wells:
for eachrow in rows:
for eachcol in columns:
templateMessage_ad = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}}",
"pipeline": posixpath.join(batchpath, batchfile),
"output": outpath,
"input": inputpath,
"data_file": posixpath.join(batchpath, batchfile),
}
assaydevqueue.scheduleBatch(templateMessage_ad)
else:
for eachwell in wells:
templateMessage_ad = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{eachcol:{well_format}}",
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell}",
"pipeline": posixpath.join(batchpath, batchfile),
"output": outpath,
"input": inputpath,
Expand All @@ -336,11 +435,26 @@ def run_batch_general(
if not csvname:
csvname = "load_data_with_illum.csv"
for plate in platelist:
for eachrow in rows:
for eachcol in columns:
if not wells:
for eachrow in rows:
for eachcol in columns:
for eachsite in sites:
templateMessage_analysis = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}},Metadata_Site={str(eachsite)}",
"pipeline": posixpath.join(pipelinepath, pipeline),
"output": outpath,
"output_structure": outputstructure,
"input": inputpath,
"data_file": posixpath.join(
datafilepath, plate, csvname
),
}
analysisqueue.scheduleBatch(templateMessage_analysis)
else:
for eachwell in wells:
for eachsite in sites:
templateMessage_analysis = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{eachcol:{well_format}},Metadata_Site={str(eachsite)}",
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell},Metadata_Site={str(eachsite)}",
"pipeline": posixpath.join(pipelinepath, pipeline),
"output": outpath,
"output_structure": outputstructure,
Expand All @@ -354,11 +468,24 @@ def run_batch_general(
if not batchfile:
batchfile = "Batch_data_analysis.h5"
for plate in platelist:
for eachrow in rows:
for eachcol in columns:
if not wells:
for eachrow in rows:
for eachcol in columns:
for eachsite in sites:
templateMessage_analysis = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{int(eachcol):{well_format}},Metadata_Site={str(eachsite)}",
"pipeline": posixpath.join(batchpath, batchfile),
"output": outpath,
"output_structure": outputstructure,
"input": inputpath,
"data_file": posixpath.join(batchpath, batchfile),
}
analysisqueue.scheduleBatch(templateMessage_analysis)
else:
for eachwell in wells:
for eachsite in sites:
templateMessage_analysis = {
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachrow}{eachcol:{well_format}},Metadata_Site={str(eachsite)}",
"Metadata": f"Metadata_Plate={plate},Metadata_Well={eachwell},Metadata_Site={str(eachsite)}",
"pipeline": posixpath.join(batchpath, batchfile),
"output": outpath,
"output_structure": outputstructure,
Expand Down Expand Up @@ -414,6 +541,13 @@ def run_batch_general(
default="1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24",
help="List of rows to process",
)
parser.add_argument(
"--wells",
dest="wells",
type=lambda s: list(s.split(",")),
default="",
help="Explicit list of rows to process. Will overwrite --rows and --columns.",
)
parser.add_argument(
"--sites",
dest="sites",
Expand Down Expand Up @@ -501,6 +635,7 @@ def run_batch_general(
plate_format=args.plate_format,
rows=args.rows,
columns=args.columns,
wells=args.wells,
sites=args.sites,
well_digit_pad=args.well_digit_pad,
pipeline=args.pipeline,
Expand Down