diff --git a/bin/postprocess2 b/bin/postprocess2 deleted file mode 100755 index 19219be743..0000000000 --- a/bin/postprocess2 +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -. `dirname $0`/../libexec/env.sh - -export MALLOC_ARENA_MAX=1 - -java ${JAVA_OPTS-} -Xmx768m -Xms768m -XX:+UseSerialGC \ - -cp ${COATJAVA_CLASSPATH:-''} \ - org.jlab.detector.serial.PostProcessor \ - $* diff --git a/common-tools/clara-io/pom.xml b/common-tools/clara-io/pom.xml index 1c3a50f9b6..e192efa6b7 100644 --- a/common-tools/clara-io/pom.xml +++ b/common-tools/clara-io/pom.xml @@ -48,12 +48,6 @@ 14.2.1-SNAPSHOT - - org.jlab.clas - clas-utils - 14.2.1-SNAPSHOT - - org.apache.commons commons-text diff --git a/common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java b/common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java index c262aa3d2b..600708b484 100644 --- a/common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java +++ b/common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java @@ -7,7 +7,6 @@ import org.jlab.detector.calib.utils.ConstantsManager; import org.jlab.detector.helicity.HelicitySequenceDelayed; import org.jlab.detector.serial.SerialHoncho; -import org.jlab.detector.serial.PostProcessor; import org.jlab.jnp.hipo4.data.Bank; import org.jlab.jnp.hipo4.data.Event; import org.jlab.jnp.hipo4.data.SchemaFactory; @@ -32,7 +31,6 @@ public class Clas12Writer extends HipoToHipoWriter { Bank runConfig; ConstantsManager conman; SchemaFactory fullSchema; - boolean postprocess; HipoWriterSorted paraWriter; long paraCount; @@ -46,7 +44,6 @@ private void init(JSONObject opts) { runConfig = new Bank(fullSchema.getSchema("RUN::config")); conman = new ConstantsManager(); conman.init("/runcontrol/hwp","/runcontrol/helicity"); - postprocess = opts.optBoolean("postprocess", false); paraTriggerMask = opts.optLong("paraTriggerMask", 0); paraTriggerPrescale = opts.optInt("paraTriggerPrescale", 0); if (opts.has("variation")) conman.setVariation(opts.getString("variation")); @@ -96,7 +93,6 @@ void writeRaw(Event physics, Event tagged) { protected void closeWriter() { serial.closure(writer); super.closeWriter(); - if (postprocess) postprocess(); serial.clear(); if (paraTriggerMask > 0) paraWriter.close(); } @@ -118,26 +114,4 @@ private int getRunNumber() { return 0; } - /** - * Copy helicity/charge tag-1 information to all events. - */ - private void postprocess() { - int d = conman.getConstants(getRunNumber(), "/runcontrol/helicity").getIntValue("delay",0,0,0); - HelicitySequenceDelayed helicity = new HelicitySequenceDelayed(d); - helicity.addStream(serial.getHelicities()); - PostProcessor p = new PostProcessor(List.of(filename), fullSchema, helicity, serial.getScalers()); - HipoReader r = new HipoReader(); - r.open(filename); - Event e = new Event(); - writer.open("pp_"+filename); - while (r.hasNext()) { - r.nextEvent(e); - p.processEvent(e); - HipoToHipoWriter.writeEvent(writer, e, schemaBankList); - } - writer.close(); - new File(filename).delete(); - new File("pp_"+filename).renameTo(new File(filename)); - } - } diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Tag1ToEvent.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Tag1ToEvent.java index b3293962f2..80cbbee22a 100644 --- a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Tag1ToEvent.java +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Tag1ToEvent.java @@ -1,6 +1,6 @@ package org.jlab.analysis.postprocess; -import org.jlab.detector.serial.PostProcessor; +import java.util.List; import java.util.TreeMap; import java.util.logging.Logger; import org.jlab.clas.reco.ReconstructionEngine; @@ -35,6 +35,31 @@ public class Tag1ToEvent { static final Logger LOGGER = Logger.getLogger(Tag1ToEvent.class.getName()); + /** + * Load the mapping from event number to unix time + * @param schema + * @param files + * @return map + */ + public static TreeMap getEventUnixMap(SchemaFactory schema, List files) { + TreeMap m = new TreeMap<>(); + Event e = new Event(); + Bank b = schema.getBank("RUN::unix");//new Bank(schema.getSchema("RUN::unix")); + for (String f : files) { + HipoReader r = new HipoReader(); + r.setTags(1); + r.open(f); + while (r.hasNext()) { + r.nextEvent(e); + e.read(b); + int size = b.getRows(); + for (int i=0; i eventUnix = PostProcessor.getEventUnixMap(schema, parser.getInputList()); + TreeMap eventUnix = getEventUnixMap(schema, parser.getInputList()); // Loop over the input HIPO files: LOGGER.info("\n>>> Starting post-processing ...\n"); diff --git a/common-tools/clas-detector/src/main/java/org/jlab/detector/serial/PostProcessor.java b/common-tools/clas-detector/src/main/java/org/jlab/detector/serial/PostProcessor.java deleted file mode 100644 index 0d0f1b96bc..0000000000 --- a/common-tools/clas-detector/src/main/java/org/jlab/detector/serial/PostProcessor.java +++ /dev/null @@ -1,277 +0,0 @@ -package org.jlab.detector.serial; - -import java.util.List; -import java.util.TreeMap; - -import org.jlab.jnp.hipo4.data.Bank; -import org.jlab.jnp.hipo4.data.Event; -import org.jlab.jnp.hipo4.data.SchemaFactory; -import org.jlab.jnp.hipo4.io.HipoReader; -import org.jlab.io.base.DataBank; -import org.jlab.io.base.DataEvent; -import org.jlab.io.hipo.HipoDataBank; - -import org.jlab.detector.calib.utils.ConstantsManager; -import org.jlab.detector.scalers.DaqScalers; -import org.jlab.detector.scalers.DaqScalersSequence; -import org.jlab.detector.helicity.HelicityBit; -import org.jlab.detector.helicity.HelicitySequenceDelayed; -import org.jlab.jnp.hipo4.io.HipoWriterSorted; -import org.jlab.utils.options.OptionParser; -import org.jlab.utils.system.ClasUtilsFile; - -/** - * - * @author baltzell - */ -public class PostProcessor { - - public static final String CCDB_TABLES[] = {"/runcontrol/fcup","/runcontrol/slm", - "/runcontrol/helicity","/daq/config/scalers/dsc1","/runcontrol/hwp"}; - - private Bank runConfig = null; - private Bank recEvent = null; - private ConstantsManager conman = null; - private SchemaFactory schemaFactory = null; - private DaqScalersSequence chargeSequence = null; - private HelicitySequenceDelayed helicitySequence = null; - private TreeMap eventUnix = null; - - public PostProcessor(List files, boolean restream, boolean rebuild) { - HipoReader r = new HipoReader(); - r.open(files.get(0)); - schemaFactory = r.getSchemaFactory(); - r.close(); - runConfig = new Bank(schemaFactory.getSchema("RUN::config")); - recEvent = new Bank(schemaFactory.getSchema("REC::Event")); - conman = new ConstantsManager(); - conman.init(CCDB_TABLES); - helicitySequence = SerialUtil.getHelicity(files, schemaFactory, restream, conman); - if (rebuild) chargeSequence = DaqScalersSequence.rebuildSequence(1, conman, files); - else chargeSequence = DaqScalersSequence.readSequence(files); - eventUnix = getEventUnixMap(schemaFactory, files); - } - - public PostProcessor(List files, SchemaFactory schema, HelicitySequenceDelayed h, DaqScalersSequence s) { - schemaFactory = schema; - helicitySequence = h; - chargeSequence = s; - runConfig = new Bank(schemaFactory.getSchema("RUN::config")); - recEvent = new Bank(schemaFactory.getSchema("REC::Event")); - conman = new ConstantsManager(); - conman.init(CCDB_TABLES); - eventUnix = getEventUnixMap(schemaFactory, files); - } - - /** - * Load the mapping from event number to unix time - * @param schema - * @param files - * @return map - */ - public static TreeMap getEventUnixMap(SchemaFactory schema, List files) { - TreeMap m = new TreeMap<>(); - Event e = new Event(); - Bank b = schema.getBank("RUN::unix");//new Bank(schema.getSchema("RUN::unix")); - for (String f : files) { - HipoReader r = new HipoReader(); - r.setTags(1); - r.open(f); - while (r.hasNext()) { - r.nextEvent(e); - e.read(b); - int size = b.getRows(); - for (int i=0; i0) { - event.removeBank("HEL::scaler"); - SerialUtil.assignScalerHelicity(runcfg.getLong("timestamp",0), ((HipoDataBank)helScaler).getBank(), helicitySequence); - event.appendBank(helScaler); - } - } - - /** - * Modify REC::Event/HEL::scaler with the delay-corrected helicity - * @param event - * @param runcfg - * @param recevt - */ - private void processEventHelicity(Event event, Bank runcfg, Bank recevt) { - HelicityBit hb = helicitySequence.search(runcfg.getLong("timestamp", 0)); - HelicityBit hbraw = helicitySequence.getHalfWavePlate() ? HelicityBit.getFlipped(hb) : hb; - recevt.putByte("helicity",0,hb.value()); - recevt.putByte("helicityRaw",0,hbraw.value()); - Bank helScaler = new Bank(schemaFactory.getSchema("HEL::scaler")); - event.read(helScaler); - if (helScaler.getRows()>0) { - event.remove(schemaFactory.getSchema("HEL::scaler")); - SerialUtil.assignScalerHelicity(runcfg.getLong("timestamp",0), helScaler, helicitySequence); - event.write(helScaler); - } - } - - /** - * Modify REC::Event for beam charge and livetime - * @param runcfg - * @param recevt - */ - private void processEventScalers(DataBank runcfg, DataBank recevt) { - DaqScalers ds = chargeSequence.get(runcfg.getLong("timestamp", 0)); - if (ds != null) { - recevt.setFloat("beamCharge",0, (float) ds.dsc2.getBeamChargeGated()); - recevt.setDouble("liveTime",0,ds.dsc2.getLivetime()); - } - } - - /** - * Modify REC::Event for beam charge and livetime - * @param runcfg - * @param recevt - */ - private void processEventScalers(Bank runcfg, Bank recevt) { - DaqScalers ds = chargeSequence.get(runcfg.getLong("timestamp", 0)); - if (ds != null) { - recevt.putFloat("beamCharge",0, (float) ds.dsc2.getBeamChargeGated()); - recevt.putDouble("liveTime",0,ds.dsc2.getLivetime()); - } - } - - /** - * Modify RUN::config's unixtime and update the event - * @param event - * @param runcfg - */ - private void processEventUnix(Event event, Bank runcfg) { - if (runcfg.getRows() > 0) { - Integer key = eventUnix.floorKey(runcfg.getInt("event",0)); - if (key != null) { - Integer unix = eventUnix.get(key); - if (unix != null) { - event.remove(runcfg.getSchema()); - runcfg.putInt("unixtime", 0, unix); - event.write(runcfg); - } - } - } - } - - /** - * Modify RUN::config's unixtime and update the event - * @param event - * @param runcfg - */ - private void processEventUnix(DataEvent event, DataBank runcfg) { - if (runcfg.rows() > 0) { - Integer key = eventUnix.floorKey(runcfg.getInt("event",0)); - if (key != null) { - Integer unix = eventUnix.get(key); - if (unix != null) { - event.removeBank(runcfg.getDescriptor().getName()); - runcfg.setInt("unixtime", 0, unix); - event.appendBank(runcfg); - } - } - } - } - - /** - * Postprocess one event - * @param event - */ - public void processEvent(DataEvent event) { - if (event.hasBank("RUN::config")) { - DataBank runcfg = event.getBank("RUN::config"); - if (runcfg.rows() > 0) { - processEventUnix(event, runcfg); - if (event.hasBank("REC::Event")) { - DataBank recevt = event.getBank("REC::Event"); - if (recevt.rows() > 0) { - event.removeBank("REC::Event"); - if (helicitySequence != null) processEventHelicity(event, runcfg, recevt); - if (chargeSequence != null) processEventScalers(runcfg, recevt); - event.appendBank(recevt); - } - } - } - } - } - - /** - * Postprocess one event - * @param event - */ - public void processEvent(Event event) { - event.read(runConfig); - event.read(recEvent); - if (runConfig.getRows() > 0) { - processEventUnix(event, runConfig); - if (recEvent.getRows() > 0) { - event.remove(recEvent.getSchema()); - if (helicitySequence != null) processEventHelicity(event, runConfig, recEvent); - if (chargeSequence != null) processEventScalers(runConfig, recEvent); - event.write(recEvent); - } - } - } - - /** - * The "postprocess" program. - * @param args - */ - public static void main(String args[]) { - - OptionParser o = new OptionParser("postprocess"); - o.addOption("-f","0","reflip: rebuild the HEL::flip bank"); - o.addOption("-c","0","recharge: rebuild the RUN/HEL::scaler banks"); - o.addOption("-o",null,"merged output file path"); - o.setRequiresInputList(true); - o.parse(args); - - boolean restream = !o.getOption("-f").isDefault(); - boolean rebuild = !o.getOption("-c").isDefault(); - - PostProcessor post = new PostProcessor(o.getInputList(), restream, rebuild); - - HipoWriterSorted writer = null; - - if (!o.getOption("-o").isDefault()) { - writer = new HipoWriterSorted(); - SchemaFactory schema = writer.getSchemaFactory(); - schema.initFromDirectory(ClasUtilsFile.getResourceDir("CLAS12DIR", "etc/bankdefs/hipo4")); - writer.setCompressionType(2); - writer.open(o.getOption("-o").stringValue()); - } - - for (String f : o.getInputList()) { - HipoReader reader = new HipoReader(); - reader.open(f); - Event event = new Event(); - while (reader.hasNext()) { - reader.nextEvent(event); - post.processEvent(event); - if (writer != null) writer.addEvent(event); - } - reader.close(); - } - - if (writer != null) writer.close(); - } - -}