Skip to content

Commit 18cac58

Browse files
baltzellbaltzell
authored
feat: add unix-time post-processing (JeffersonLab#1281)
* add unix time to postproc * make it static * propagate to old post-processor * finish the old new postprocessor * add new postprocessor * remove constructor * cleanup constructors * process all events * cleanup/bugfix * bugfix * Revert "bugfix" This reverts commit ad7484f. * Revert "cleanup/bugfix" This reverts commit 198f395. * Revert "process all events" This reverts commit 7a5b38a. * Revert "cleanup constructors" This reverts commit 9b6fd5b. * Revert "remove constructor" This reverts commit 3a53b39. * Revert "add new postprocessor" This reverts commit 89902ed. * Revert "finish the old new postprocessor" This reverts commit 6735592. * add null checks * fix javadoc, add missing null check
1 parent 2ea8db3 commit 18cac58

3 files changed

Lines changed: 155 additions & 56 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,8 @@ jobs:
225225
- name: run test
226226
run: |
227227
ls -lhtr
228-
./coatjava/bin/decoder -l FINE -n 10000 -o dog.hipo ./clas_018779.evio.00001
228+
./coatjava/bin/decoder -l FINE -n 10000 -o dec.hipo clas_018779.evio.00001
229+
./coatjava/bin/postprocess -d 1 -q 1 -o pp.hipo dec.hipo
229230
230231
test_clara:
231232
needs: [ build, download_test_data ]

‎common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java‎

Lines changed: 135 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import java.util.HashMap;
99
import java.util.List;
1010
import java.util.Map;
11+
import java.util.TreeMap;
1112

1213
import org.jlab.jnp.hipo4.data.Bank;
1314
import org.jlab.jnp.hipo4.data.Event;
@@ -35,10 +36,13 @@ public class Processor {
3536

3637
private final String outputPrefix = "tmp_";
3738

39+
private Bank runConfig = null;
40+
private Bank recEvent = null;
3841
private ConstantsManager conman = null;
3942
private SchemaFactory schemaFactory = null;
4043
private DaqScalersSequence chargeSequence = null;
4144
private HelicitySequenceDelayed helicitySequence = null;
45+
private TreeMap<Integer,Integer> eventUnix = null;
4246

4347
public Processor(File file, boolean restream, boolean rebuild) {
4448
configure(Arrays.asList(file.getAbsolutePath()), restream, rebuild);
@@ -58,19 +62,24 @@ public Processor(SchemaFactory schema, HelicitySequenceDelayed h, DaqScalersSequ
5862
schemaFactory = schema;
5963
helicitySequence = h;
6064
chargeSequence = s;
65+
runConfig = new Bank(schemaFactory.getSchema("RUN::config"));
66+
recEvent = new Bank(schemaFactory.getSchema("REC::Event"));
6167
}
6268

6369
private void configure(List<String> preloadFiles, boolean restream, boolean rebuild) {
6470
if (!preloadFiles.isEmpty()) {
6571
HipoReader r = new HipoReader();
6672
r.open(preloadFiles.get(0));
73+
schemaFactory = r.getSchemaFactory();
74+
r.close();
75+
runConfig = new Bank(schemaFactory.getSchema("RUN::config"));
76+
recEvent = new Bank(schemaFactory.getSchema("REC::Event"));
6777
conman = new ConstantsManager();
6878
conman.init(CCDB_TABLES);
69-
schemaFactory = r.getSchemaFactory();
7079
helicitySequence = Util.getHelicity(preloadFiles, schemaFactory, restream, conman);
7180
if (rebuild) chargeSequence = DaqScalersSequence.rebuildSequence(1, conman, preloadFiles);
7281
else chargeSequence = DaqScalersSequence.readSequence(preloadFiles);
73-
r.close();
82+
eventUnix = getEventUnixMap(schemaFactory, preloadFiles);
7483
}
7584
}
7685

@@ -92,117 +101,188 @@ private static List<String> findPreloadFiles(String dir, String glob) {
92101
return ret;
93102
}
94103

104+
/**
105+
* Load the mapping from event number to unix time
106+
* @param schema
107+
* @param files
108+
* @return map
109+
*/
110+
public static TreeMap<Integer,Integer> getEventUnixMap(SchemaFactory schema, List<String> files) {
111+
Bank unix = new Bank(schema.getSchema("RUN::unix"));
112+
TreeMap<Integer,Integer> m = new TreeMap<>();
113+
Event e = new Event();
114+
for (String f : files) {
115+
HipoReader r = new HipoReader();
116+
r.setTags(1);
117+
r.open(f);
118+
while (r.hasNext()) {
119+
r.nextEvent(e);
120+
e.read(unix);
121+
int size = unix.getRows();
122+
for (int i=0; i<size; i++) {
123+
m.put(unix.getInt("event",i), unix.getInt("unixtime",i));
124+
}
125+
}
126+
r.close();
127+
}
128+
return m;
129+
}
130+
95131
/**
96132
* Modify REC::Event/HEL::scaler with the delay-corrected helicity
97133
* @param event
98-
* @param runConfig
99-
* @param recEvent
134+
* @param runcfg
135+
* @param recevt
100136
*/
101-
private void processEventHelicity(DataEvent event, DataBank runConfig, DataBank recEvent) {
102-
HelicityBit hb = helicitySequence.search(runConfig.getLong("timestamp", 0));
137+
private void processEventHelicity(DataEvent event, DataBank runcfg, DataBank recevt) {
138+
HelicityBit hb = helicitySequence.search(runcfg.getLong("timestamp", 0));
103139
HelicityBit hbraw = helicitySequence.getHalfWavePlate() ? HelicityBit.getFlipped(hb) : hb;
104-
recEvent.setByte("helicity",0,hb.value());
105-
recEvent.setByte("helicityRaw",0,hbraw.value());
140+
recevt.setByte("helicity",0,hb.value());
141+
recevt.setByte("helicityRaw",0,hbraw.value());
106142
DataBank helScaler = event.getBank("HEL::scaler");
107143
if (helScaler.rows()>0) {
108144
event.removeBank("HEL::scaler");
109-
Util.assignScalerHelicity(runConfig.getLong("timestamp",0), ((HipoDataBank)helScaler).getBank(), helicitySequence);
145+
Util.assignScalerHelicity(runcfg.getLong("timestamp",0), ((HipoDataBank)helScaler).getBank(), helicitySequence);
110146
event.appendBank(helScaler);
111147
}
112148
}
113149

114150
/**
115151
* Modify REC::Event/HEL::scaler with the delay-corrected helicity
116152
* @param event
117-
* @param runConfig
118-
* @param recEvent
153+
* @param runcfg
154+
* @param recevt
119155
*/
120-
private void processEventHelicity(Event event, Bank runConfig, Bank recEvent) {
121-
HelicityBit hb = helicitySequence.search(runConfig.getLong("timestamp", 0));
156+
private void processEventHelicity(Event event, Bank runcfg, Bank recevt) {
157+
HelicityBit hb = helicitySequence.search(runcfg.getLong("timestamp", 0));
122158
HelicityBit hbraw = helicitySequence.getHalfWavePlate() ? HelicityBit.getFlipped(hb) : hb;
123-
recEvent.setByte("helicity",0,hb.value());
124-
recEvent.setByte("helicityRaw",0,hbraw.value());
159+
recevt.setByte("helicity",0,hb.value());
160+
recevt.setByte("helicityRaw",0,hbraw.value());
125161
Bank helScaler = new Bank(schemaFactory.getSchema("HEL::scaler"));
126162
event.read(helScaler);
127163
if (helScaler.getRows()>0) {
128164
event.remove(schemaFactory.getSchema("HEL::scaler"));
129-
Util.assignScalerHelicity(runConfig.getLong("timestamp",0), helScaler, helicitySequence);
165+
Util.assignScalerHelicity(runcfg.getLong("timestamp",0), helScaler, helicitySequence);
130166
event.write(helScaler);
131167
}
132168
}
133169

134170
/**
135171
* Modify REC::Event for beam charge and livetime
136-
* @param runConfig
137-
* @param recEvent
172+
* @param runcfg
173+
* @param recevt
138174
*/
139-
private void processEventScalers(DataBank runConfig, DataBank recEvent) {
140-
DaqScalers ds = chargeSequence.get(runConfig.getLong("timestamp", 0));
175+
private void processEventScalers(DataBank runcfg, DataBank recevt) {
176+
DaqScalers ds = chargeSequence.get(runcfg.getLong("timestamp", 0));
141177
if (ds != null) {
142-
recEvent.setFloat("beamCharge",0, (float) ds.dsc2.getBeamChargeGated());
143-
recEvent.setDouble("liveTime",0,ds.dsc2.getLivetime());
178+
recevt.setFloat("beamCharge",0, (float) ds.dsc2.getBeamChargeGated());
179+
recevt.setDouble("liveTime",0,ds.dsc2.getLivetime());
144180
}
145181
}
146182

147183
/**
148184
* Modify REC::Event for beam charge and livetime
149-
* @param runConfig
150-
* @param recEvent
185+
* @param runcfg
186+
* @param recevt
151187
*/
152-
private void processEventScalers(Bank runConfig, Bank recEvent) {
153-
DaqScalers ds = chargeSequence.get(runConfig.getLong("timestamp", 0));
188+
private void processEventScalers(Bank runcfg, Bank recevt) {
189+
DaqScalers ds = chargeSequence.get(runcfg.getLong("timestamp", 0));
154190
if (ds != null) {
155-
recEvent.putFloat("beamCharge",0, (float) ds.dsc2.getBeamChargeGated());
156-
recEvent.putDouble("liveTime",0,ds.dsc2.getLivetime());
191+
recevt.putFloat("beamCharge",0, (float) ds.dsc2.getBeamChargeGated());
192+
recevt.putDouble("liveTime",0,ds.dsc2.getLivetime());
193+
}
194+
}
195+
196+
/**
197+
* Modify RUN::config's unixtime and update the event
198+
* @param event
199+
* @param runcfg
200+
*/
201+
private void processEventUnix(Event event, Bank runcfg) {
202+
if (runcfg.getRows() > 0) {
203+
Integer key = eventUnix.floorKey(runcfg.getInt("event",0));
204+
if (key != null) {
205+
Integer unix = eventUnix.get(key);
206+
if (unix != null) {
207+
event.remove(runcfg.getSchema());
208+
runcfg.putInt("unixtime", 0, unix);
209+
event.write(runcfg);
210+
}
211+
}
212+
}
213+
}
214+
215+
/**
216+
* Modify RUN::config's unixtime and update the event
217+
* @param event
218+
* @param runcfg
219+
*/
220+
private void processEventUnix(DataEvent event, DataBank runcfg) {
221+
if (runcfg.rows() > 0) {
222+
Integer key = eventUnix.floorKey(runcfg.getInt("event",0));
223+
if (key != null) {
224+
Integer unix = eventUnix.get(key);
225+
if (unix != null) {
226+
event.removeBank(runcfg.getDescriptor().getName());
227+
runcfg.setInt("unixtime", 0, unix);
228+
event.appendBank(runcfg);
229+
}
230+
}
157231
}
158232
}
159233

160234
/**
161235
* Postprocess one event
162-
* @param e
236+
* @param event
163237
*/
164-
public void processEvent(DataEvent e) {
165-
if (!e.hasBank("RUN::config")) return;
166-
if (!e.hasBank("REC::Event")) return;
167-
DataBank runConfig = e.getBank("RUN::config");
168-
DataBank recEvent = e.getBank("REC::Event");
169-
if (runConfig.rows()<1 || recEvent.rows()<1) return;
170-
e.removeBank("REC::Event");
171-
if (helicitySequence != null) processEventHelicity(e, runConfig, recEvent);
172-
if (chargeSequence != null) processEventScalers(runConfig, recEvent);
173-
e.appendBank(recEvent);
238+
public void processEvent(DataEvent event) {
239+
if (event.hasBank("RUN::config")) {
240+
DataBank runcfg = event.getBank("RUN::config");
241+
if (runcfg.rows() > 0) {
242+
processEventUnix(event, runcfg);
243+
if (event.hasBank("REC::Event")) {
244+
DataBank recevt = event.getBank("REC::Event");
245+
if (recevt.rows() > 0) {
246+
event.removeBank("REC::Event");
247+
if (helicitySequence != null) processEventHelicity(event, runcfg, recevt);
248+
if (chargeSequence != null) processEventScalers(runcfg, recevt);
249+
event.appendBank(recevt);
250+
}
251+
}
252+
}
253+
}
174254
}
175255

176256
/**
177257
* Postprocess one event
178-
* @param e
258+
* @param event
179259
*/
180-
public void processEvent(Event e) {
181-
if (!e.hasBank(schemaFactory.getSchema("RUN::config"))) return;
182-
if (!e.hasBank(schemaFactory.getSchema("REC::Event"))) return;
183-
Bank runConfig = new Bank(schemaFactory.getSchema("RUN::config"));
184-
Bank recEvent = new Bank(schemaFactory.getSchema("REC::Event"));
185-
e.read(runConfig);
186-
e.read(recEvent);
187-
if (runConfig.getRows()<1 || recEvent.getRows()<1) return;
188-
e.remove(schemaFactory.getSchema("REC::Event"));
189-
if (helicitySequence != null) processEventHelicity(e, runConfig, recEvent);
190-
if (chargeSequence != null) processEventScalers(runConfig, recEvent);
191-
e.write(recEvent);
260+
public void processEvent(Event event) {
261+
event.read(runConfig);
262+
event.read(recEvent);
263+
if (runConfig.getRows() > 0) {
264+
processEventUnix(event, runConfig);
265+
if (recEvent.getRows() > 0) {
266+
event.remove(recEvent.getSchema());
267+
if (helicitySequence != null) processEventHelicity(event, runConfig, recEvent);
268+
if (chargeSequence != null) processEventScalers(runConfig, recEvent);
269+
event.write(recEvent);
270+
}
271+
}
192272
}
193273

194274
/**
195275
* Create rebuilt files from preload files.
196-
* @param preloadFiles
276+
* @param files
197277
* @return map of rebuilt:preload files
198278
*/
199-
private Map<String,String> rebuild(String dir, List<String> preloadFiles) {
279+
private Map<String,String> rebuild(String dir, List<String> files) {
200280
File d = new File(dir);
201281
if (!d.canWrite()) {
202282
throw new RuntimeException("No write permissions on "+dir);
203283
}
204284
Map<String,String> rebuiltFiles = new HashMap<>();
205-
for (String preloadFile : preloadFiles) {
285+
for (String preloadFile : files) {
206286
String rebuiltFile = dir+"/"+outputPrefix+preloadFile.replace(dir+"/","");
207287
Util.rebuildScalers(conman, preloadFile, rebuiltFile);
208288
rebuiltFiles.put(rebuiltFile,preloadFile);

‎common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Tag1ToEvent.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.jlab.analysis.postprocess;
22

3+
import java.util.TreeMap;
34
import java.util.logging.Logger;
45
import org.jlab.clas.reco.ReconstructionEngine;
56
import org.jlab.detector.calib.utils.ConstantsManager;
@@ -100,6 +101,9 @@ public static void main(String[] args) {
100101
helSeq.initialize(parser.getInputList());
101102
}
102103

104+
// Initialize the unix-event map:
105+
TreeMap<Integer,Integer> eventUnix = Processor.getEventUnixMap(schema, parser.getInputList());
106+
103107
// Loop over the input HIPO files:
104108
LOGGER.info("\n>>> Starting post-processing ...\n");
105109
for (String filename : parser.getInputList()) {
@@ -147,6 +151,20 @@ public static void main(String[] args) {
147151
event.write(recEventBank);
148152
event.write(helScalerBank);
149153

154+
// Update RUN::config.unixtime:
155+
if (runConfigBank.getRows() > 0) {
156+
int evno = runConfigBank.getInt("event", 0);
157+
Integer key = eventUnix.floorKey(evno);
158+
if (key != null) {
159+
Integer unix = eventUnix.get(key);
160+
if (unix != null) {
161+
event.remove(runConfigBank.getSchema());
162+
runConfigBank.putInt("unixtime", 0, unix);
163+
event.write(runConfigBank);
164+
}
165+
}
166+
}
167+
150168
// Write out the original event:
151169
writer.addEvent(event, event.getEventTag());
152170

0 commit comments

Comments
 (0)