recon-mutil

.github/workflows/ci.yml

@@ -247,12 +247,10 @@ jobs:
           tar xzf clara.tar.gz
           tar xzf coatjava.tar.gz
       - run: ls
-      - name: run test
-        run: ./bin/run-clara -y ./etc/services/rgd-clarode.yml -t 4 -n 500 -c ./clara -o ./tmp ./clas_018779.evio.00001
-      - name: ls tmp
-        run: ls -lhtr tmp
-      - name: rename
-        run: mv -v tmp/rec_clas_018779.evio.00001.hipo rec.hipo
+      - name: run clara
+        run: ./coatjava/bin/run-clara -y ./etc/services/rgd-clarode.yml -t 4 -n 100 -c ./clara -o ./tmp ./clas_018779.evio.00001
+      - name: run mutil
+        run: ./coatjava/bin/recon-mutil -y ./etc/services/rgd-clarode.yml -t 4 -o rec.hipo ./clas_018779.evio.00001
       - uses: actions/upload-artifact@v7
         with:
           name: test_clara_result

.gitignore

@@ -58,3 +58,6 @@ hs_err_pid*
 
 bin/evio2hipotest
 export.sh
+
+*.txt
+

.gitlab-ci.yml

@@ -175,13 +175,24 @@ clara:
     - run-clara -v -c $CLARA_HOME -t 4 -y ./etc/services/rgd-clarode.yml -n 30 -o out $EVIOFILE
     - mv out/rec_$EVIOFILE.hipo claroded.hipo
 
+recon-mutil:
+  stage: test
+  needs: [build,download]
+  dependencies: [build,download]
+  script:
+    - tar -xzf coatjava.tar.gz
+    - recon-mutil -t 4 -y ./etc/services/rgd-clarode.yml -n 30 -o rec_$EVIOFILE.hipo $EVIOFILE
+
 profile:
   extends: .clon
   allow_failure: true
   script:
-    - ./clara/plugins/clas12/libexec/profile -e cpu   run-clara -c ./clara -y ./etc/services/rgd-clarode.yml -o tmp1 -t 16 -l $EVIOFILE
-    - ./clara/plugins/clas12/libexec/profile -e alloc run-clara -c ./clara -y ./etc/services/rgd-clarode.yml -o tmp2 -t 16 -l $EVIOFILE
-    - ./clara/plugins/clas12/libexec/profile -e lock  run-clara -c ./clara -y ./etc/services/rgd-clarode.yml -o tmp3 -t 16 -l $EVIOFILE
+    - ./clara/plugins/clas12/libexec/profile -e cpu   -o _clara run-clara -c ./clara -y ./etc/services/rgd-clarode.yml -o tmp1 -t 16 -l $EVIOFILE
+    - ./clara/plugins/clas12/libexec/profile -e alloc -o _clara run-clara -c ./clara -y ./etc/services/rgd-clarode.yml -o tmp2 -t 16 -l $EVIOFILE
+    - ./clara/plugins/clas12/libexec/profile -e lock  -o _clara run-clara -c ./clara -y ./etc/services/rgd-clarode.yml -o tmp3 -t 16 -l $EVIOFILE
+    - ./clara/plugins/clas12/libexec/profile -e cpu   -o _mutil recon-mutil -y ./etc/services/rgd-clarode.yml -o mutil1.hipo -t 16 -l SEVERE $EVIOFILE
+    - ./clara/plugins/clas12/libexec/profile -e alloc -o _mutil recon-mutil -y ./etc/services/rgd-clarode.yml -o mutil2.hipo -t 16 -l SEVERE $EVIOFILE
+    - ./clara/plugins/clas12/libexec/profile -e lock  -o _mutil recon-mutil -y ./etc/services/rgd-clarode.yml -o mutil3.hipo -t 16 -l SEVERE $EVIOFILE
   artifacts:
     when: always
     expire_in: 1 week
@@ -197,7 +208,7 @@ scaling:
     when: always
     expire_in: 1 week
     paths:
-      - scaling.svg
+      - scaling-clara.svg
 
 pages:
   stage: deploy
@@ -207,7 +218,7 @@ pages:
     - cp -r pages public
     - mkdir public/perf
     - cp asprof*.html public/perf
-    - cp scaling.svg public/perf
+    - cp scaling*.svg public/perf
   pages: true
   artifacts:
     expire_in: 1 week

bin/reco-mutil

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. `dirname $0`/../libexec/env.sh
+
+split_cli $@
+
+export MALLOC_ARENA_MAX=1
+
+#threads=$(get_threads)
+#node=$(((RANDOM % 2)))
+#cpus=$(get_all_numa_cpus $node)
+#cpus=$(get_numa_cpus $node)
+#taskset -c $cpus \
+
+    java ${JAVA_OPTS-} -XX:MaxRAMPercentage=40.0 -Xms10240m ${jvm_options[@]} \
+    -cp ${COATJAVA_CLASSPATH:-''} \
+    org.jlab.clas.reco.RecoMutil \
+    ${class_options[@]}

bin/recon-mutil

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+. `dirname $0`/../libexec/env.sh
+
+split_cli $@
+
+export MALLOC_ARENA_MAX=12
+
+threads=$(get_threads)
+
+    #java ${JAVA_OPTS-} -Xmx256g -Xms10240m ${jvm_options[@]} \
+    #java ${JAVA_OPTS-} -Xms10240m -XX:+UseParallelGC ${jvm_options[@]} \
+    java ${JAVA_OPTS-} -XX:MaxRAMPercentage=40.0 -Xms10240m ${jvm_options[@]} \
+    -cp ${COATJAVA_CLASSPATH:-''} \
+    org.jlab.clas.reco.ReconMutil \
+    ${class_options[@]}

bin/recon-util

@@ -4,7 +4,7 @@
 
 split_cli $@
 
-export MALLOC_ARENA_MAX=1
+export MALLOC_ARENA_MAX=12
 
 java ${JAVA_OPTS-} -Xmx1536m -Xms1024m -XX:+UseSerialGC ${jvm_options[@]} \
     -cp ${COATJAVA_CLASSPATH:-''} \

common-tools/clara-io/pom.xml

@@ -48,12 +48,6 @@
       <version>14.2.1-SNAPSHOT</version>
     </dependency>
 
-    <dependency>
-      <groupId>org.jlab.clas</groupId>
-      <artifactId>clas-utils</artifactId>
-      <version>14.2.1-SNAPSHOT</version>
-    </dependency>
-
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-text</artifactId>

common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java

@@ -1,17 +1,13 @@
 package org.jlab.io.clara;
 
-import java.io.File;
 import java.nio.file.Path;
-import java.util.List;
 import org.jlab.clara.std.services.EventWriterException;
 import org.jlab.detector.calib.utils.ConstantsManager;
-import org.jlab.detector.helicity.HelicitySequenceDelayed;
 import org.jlab.detector.serial.SerialHoncho;
-import org.jlab.detector.serial.PostProcessor;
+import org.jlab.detector.serial.Occupancer;
 import org.jlab.jnp.hipo4.data.Bank;
 import org.jlab.jnp.hipo4.data.Event;
 import org.jlab.jnp.hipo4.data.SchemaFactory;
-import org.jlab.jnp.hipo4.io.HipoReader;
 import org.jlab.jnp.hipo4.io.HipoWriterSorted;
 import org.jlab.jnp.utils.file.FileUtils;
 import org.json.JSONObject;
@@ -21,32 +17,31 @@
  * 1. Copies certain banks on-the-fly to new tag-1 events
  * 2. Caches helicity states, scaler readouts, and unix time
  * 3. Writes HEL::flip, RUN/HEL::scaler, and RUN::unix to new tag-1 events
- * 4. Runs post-processing, writing tag-1 information to all events 
  * 5. Adds .hipo to the output filename, if necessary
  * 6. Outputs a parallel file of select trigger bits with a raw schema
  * @author baltzell
  */
 public class Clas12Writer extends HipoToHipoWriter {
 
+    Occupancer occupancer;
     SerialHoncho serial;
-    Bank runConfig;
     ConstantsManager conman;
     SchemaFactory fullSchema;
-    boolean postprocess;
+    Bank runConfig;
 
     HipoWriterSorted paraWriter;
     long paraCount;
     long paraTriggerMask;
     int paraTriggerPrescale;
 
     private void init(JSONObject opts) {
+        occupancer = new Occupancer();
         fullSchema = new SchemaFactory();
         fullSchema.initFromDirectory(FileUtils.getEnvironmentPath("CLAS12DIR","etc/bankdefs/hipo4"));
         serial = new SerialHoncho(fullSchema);
-        runConfig = new Bank(fullSchema.getSchema("RUN::config"));
         conman = new ConstantsManager();
+        runConfig = new Bank(fullSchema.getSchema("RUN::config"));
         conman.init("/runcontrol/hwp","/runcontrol/helicity");
-        postprocess = opts.optBoolean("postprocess", false);
         paraTriggerMask = opts.optLong("paraTriggerMask", 0);
         paraTriggerPrescale = opts.optInt("paraTriggerPrescale", 0);
         if (opts.has("variation")) conman.setVariation(opts.getString("variation"));
@@ -78,6 +73,7 @@ protected HipoWriterSorted createWriter(Path file, JSONObject opts) throws Event
     protected void writeEvent(Object event) throws EventWriterException {
         Event t = serial.read((Event)event);
         if (!t.isEmpty()) writer.addEvent(t, 1);
+        occupancer.process(((Event)event));
         super.writeEvent(event);
         writeRaw((Event)event, t);
     }
@@ -96,48 +92,8 @@ void writeRaw(Event physics, Event tagged) {
     protected void closeWriter() {
         serial.closure(writer);
         super.closeWriter();
-        if (postprocess) postprocess();
         serial.clear();
         if (paraTriggerMask > 0) paraWriter.close();
     }
-
-    /**
-     * Get the first valid run number from a RUN::config bank.
-     * @return run
-     */
-    private int getRunNumber() {
-        Event e = new Event();
-        HipoReader r = new HipoReader();
-        r.open(filename);
-        while (r.hasNext()) {
-            r.nextEvent(e);
-            e.read(runConfig);
-            if (runConfig.getRows()>0 && runConfig.getInt("run",0)>0)
-                return runConfig.getInt("run",0);
-        }
-        return 0;
-    }
-
-    /**
-     * Copy helicity/charge tag-1 information to all events.
-     */
-    private void postprocess() {
-        int d = conman.getConstants(getRunNumber(), "/runcontrol/helicity").getIntValue("delay",0,0,0);
-        HelicitySequenceDelayed helicity = new HelicitySequenceDelayed(d);
-        helicity.addStream(serial.getHelicities());
-        PostProcessor p = new PostProcessor(List.of(filename), fullSchema, helicity, serial.getScalers());
-        HipoReader r = new HipoReader();
-        r.open(filename);
-        Event e = new Event();
-        writer.open("pp_"+filename);
-        while (r.hasNext()) {
-            r.nextEvent(e);
-            p.processEvent(e);
-            HipoToHipoWriter.writeEvent(writer, e, schemaBankList);
-        }
-        writer.close();
-        new File(filename).delete();
-        new File("pp_"+filename).renameTo(new File(filename));
-    }
 
 }