Changeset 43

Timestamp:

Dec 5, 2011, 7:05:00 PM (13 years ago)

Author:

g7moreau

Message:

• --file -> --filecmd
• --logfile -> --logtrace
• Comment code
• Begin man for option --logtrace

File:

• trunk/oarutils/oar-parexec (modified) (18 diffs)

trunk/oarutils/oar-parexec

@@ -16 +16 @@
 use Cwd qw( getcwd );
 
-my $file    = '';
-my $logfile = '';
+my $filecmd  = '';
+my $logtrace = '';
 my $verbose;
 my $job_np = 1;
@@ -27 +27 @@
 
 Getopt::Long::GetOptions(
-   'file=s'     => \$file,
-   'logfile=s'  => \$logfile,
+   'filecmd=s'  => \$filecmd,
+   'logtrace=s' => \$logtrace,
    'verbose'    => \$verbose,
    'help'       => \$help,
@@ -38 +38 @@
    ) || pod2usage(-verbose => 0);
 pod2usage(-verbose => 2) if $help;
-pod2usage(-verbose => 2) if not -e $file;
-
+pod2usage(-verbose => 2) if not -e $filecmd;
+
+# re-run, keep trace of job already done
 my %state;
 my $log_h = IO::File->new();
-if (-e $logfile) {
-   $log_h->open("< $logfile")
-      or die "can't read log file: $!";
+if (-e $logtrace) {
+   $log_h->open("< $logtrace")
+      or die "error: can't read log file: $!";
    while (<$log_h>) {
       $state{$1} = 'start' if m/^start\s+job\s+(\d+)\s/;
@@ -51 +52 @@
    $log_h->close();
    }
-if ($logfile) {
-   $log_h->open(">> $logfile")
-      or die "can't append log file $logfile: $!";
+if ($logtrace) {
+   $log_h->open(">> $logtrace")
+      or die "error: can't append log file $logtrace: $!";
    $log_h->autoflush;
    $log_h = unblock $log_h;
    }
 
+# job to run
 my @job = ();
-open(JOB_LIST, '<', "$file") or die "can't open job file $file: $!";
+open(JOB_LIST, '<', "$filecmd") or die "error: can't open job file $filecmd: $!";
 while (<JOB_LIST>) {
    chomp;
@@ -68 +70 @@
 close JOB_LIST;
 
+# ressources available
 my @ressources = ();
 open(NODE_FILE, '<', "$nodefile")
@@ -80 +83 @@
 
 my $ressource_size = scalar(@ressources);
-die "not enought ressources jobnp $job_np > ressources $ressource_size"
+die "error: not enought ressources jobnp $job_np > ressources $ressource_size"
    if $job_np > $ressource_size;
 
@@ -96 +99 @@
 $job_todo->up for (@job);
 
+# slice of ressources for parallel job
 my $ressources = new Coro::Channel;
 for my $slot (1 .. int($ressource_size / $job_np)) {
@@ -107 +111 @@
 my %scheduled = ();
 
+# OAR checkpoint and default signal SIGUSR2
 my $oar_checkpoint = new Coro::Semaphore 0;
 $SIG{USR2} = sub {
@@ -116 +121 @@
    };
 
+# asynchrone start job block
 async {
+        JOB:
    for my $job (@job) {
       $job_num++;
 
+      # job has been already run ?
       if (exists $state{$job_num}) {
          if ($state{$job_num} eq 'start') {
-            print "warning: job $job_num was not finished, relaunching...\n"
+            print "warning: job $job_num was not clearly finished, relaunching...\n"
                if $verbose;
             }
          elsif ($state{$job_num} eq 'end') {
-            delete $state{$job_num};
+            delete $state{$job_num}; # free memory
             $job_todo->down;
-            print "warning: job $job_num already done\n" if $verbose;
+            print "warning: job $job_num already run\n" if $verbose;
             cede;
-            next;
+            next JOB;
             }
          }
 
+      # take job ressource
       my $job_ressource = $ressources->get;
 
-      last if $oar_checkpoint->count() > 0;
+      # no more launch job when OAR checkpointing
+      last JOB if $oar_checkpoint->count() > 0;
 
       my ($node_connect) = split ',', $job_ressource;
       my $fh = IO::File->new();
       my $job_pid = $fh->open("| $oarsh $node_connect >/dev/null 2>&1")
-         or die "don't start subjob: $!";
+         or die "error: can't start subjob: $!";
 
       $fh->autoflush;
@@ -155 +165 @@
       my $msg = sprintf "start job %5i / %5i at %s on node %s\n",
          $job_num, $job_pid, time, $job_ressource;
-      $log_h->print($msg) if $logfile;
+      $log_h->print($msg) if $logtrace;
       print($msg) if $verbose;
 
@@ -164 +174 @@
       my $job_nodefile = "/tmp/oar-parexec-$ENV{LOGNAME}-$job_num";
 
+     # set job environment, run it and clean
       if ($job_np > 1) {
          $fh->print("printf \""
@@ -182 +193 @@
    }
 
+# asynchrone end job block
 async {
    while () {
       for my $job_pid (keys %scheduled) {
+                        # non blocking PID test
          if (waitpid($job_pid, WNOHANG)) {
             my $msg = sprintf "end   job %5i / %5i at %s on node %s\n",
                $scheduled{$job_pid}->{num},
                $job_pid, time, $scheduled{$job_pid}->{ressource};
-            $log_h->print($msg) if $logfile;
+            $log_h->print($msg) if $logtrace;
             print($msg) if $verbose;
             close $scheduled{$job_pid}->{fh};
+            # leave ressources for another job
             $ressources->put($scheduled{$job_pid}->{ressource});
             $job_todo->down;
@@ -199 +213 @@
          }
 
+      # checkpointing ! just finishing running job and quit
       $finished->send if $oar_checkpoint->count() > 0 and scalar(keys(%scheduled)) == 0;
 
@@ -208 +223 @@
 cede;
 
+# all job have been done
 $finished->wait;
 
-$log_h->close() if $logfile;
+# close log trace file
+$log_h->close() if $logtrace;
 
 __END__
@@ -220 +237 @@
 =head1 SYNOPSIS
 
- oar-parexec --file filecommand [--verbose] [--jobnp integer] [--nodefile filenode] [--masterio basefileio] [--switchio] [--oarsh sssh]
+ oar-parexec --filecmd filecommand [--logtrace tracefile] [--verbose] [--jobnp integer] [--nodefile filenode] [--masterio basefileio] [--switchio] [--oarsh sssh]
  oar-parexec --help
 
@@ -230 +247 @@
 which define automatically theses strategics parameters...
 
-Option C<--file> is the only mandatory one.
+Option C<--filecmd> is the only mandatory one.
 
 Small job will be launch in the same folder as the master job.
@@ -254 +271 @@
 =over 12
 
-=item B<-f|--file       filecommand>
+=item B<-f|--filecmd    filecommand>
 
 File name which content job list.
+
+=item B<-l|--logtrace tracefile>
+
+File which log and trace running job.
+In case of running the same command (after crash for example),
+only job that ar not mark as done will be run again.
+Be carefful, job mark as running (start but for finish) will be run again.
+
+This option is very usefull in case of crash
+but also for checkpointing and idempotent OAR job.
 
 =item B<-v|--verbose>
@@ -308 +335 @@
 =head1 EXAMPLE
 
-Content for the job file (option C<--file>) could have:
+Content for the job file command (option C<--filecmd>) could have:
 
  - empty line