Changeset 45 for trunk/oarutils/oar-parexec

Timestamp:

Dec 6, 2011, 12:26:31 PM (12 years ago)

Author:

g7moreau

Message:

• Folder list job

File:

• trunk/oarutils/oar-parexec (modified) (15 diffs)

trunk/oarutils/oar-parexec

@@ -17 +17 @@
 
 my $filecmd  = '';
+my $dir      = '';
+my $cmd      = '';
 my $logtrace = '';
 my $verbose;
@@ -28 +30 @@
 Getopt::Long::GetOptions(
    'filecmd=s'  => \$filecmd,
+   'dir=s'      => \$dir,
+   'cmd=s'      => \$cmd,
    'logtrace=s' => \$logtrace,
    'verbose'    => \$verbose,
@@ -38 +42 @@
    ) || pod2usage(-verbose => 0);
 pod2usage(-verbose => 2) if $help;
-pod2usage(-verbose => 2) if not -e $filecmd;
+pod2usage(-verbose => 2) if not (
+ (-e "$filecmd")
+ or (-d "$dir" and $cmd ne '')
+ );
 
 # re-run, keep trace of job already done
 my %state;
 my $log_h = IO::File->new();
-if (-e $logtrace) {
+if (-e "$logtrace") {
    $log_h->open("< $logtrace")
       or die "error: can't read log file: $!";
    while (<$log_h>) {
-      $state{$1} = 'start' if m/^start\s+job\s+(\d+)\s/;
-      $state{$1} = 'end'   if m/^end\s+job\s+(\d+)\s/;
+      $state{$1} = 'start' if m/^start\s+job\s+([^\s]+)\s/;
+      $state{$1} = 'end'   if m/^end\s+job\s+([^\s]+)\s/;
       }
    $log_h->close();
@@ -61 +68 @@
 # job to run
 my @job = ();
-open(JOB_LIST, '<', "$filecmd") or die "error: can't open job file $filecmd: $!";
-while (<JOB_LIST>) {
-   chomp;
-   next if m/^#/;
-   next if m/^\s*$/;
-   push @job, $_;
-   }
-close JOB_LIST;
+if (-e "$filecmd") {
+   my $job_num = 0;
+   open(JOB_LIST, '<', "$filecmd") or die "error: can't open job file $filecmd: $!";
+   while (<JOB_LIST>) {
+      chomp;
+      next if m/^#/;
+      next if m/^\s*$/;
+      $job_num++;
+      push @job, { name => $job_num, cmd => "$_" };
+      }
+   close JOB_LIST;
+   }
+else {
+   opendir(DIR, $dir) or die "error: can't open folder $dir: $!";
+   while (my $item = readdir(DIR)) {
+      next if $item =~ m/^\./;
+      next if $item =~ m/:/;
+      next if $item =~ m/\.old$/;
+      next if $item =~ m/\.sav$/;
+      next if $item =~ m/\.bak$/;
+      next if $item =~ m/\.no$/;
+      next unless (-d "$dir/$item");
+      push @job, { name => $item, cmd => "( cd $dir/$item/; $cmd )" };
+      }
+   closedir DIR;
+   }
 
 # ressources available
@@ -97 +122 @@
 my $finished = new Coro::Signal;
 my $job_todo = new Coro::Semaphore 0;
-$job_todo->up for (@job);
+my $job_name_maxlen;
+for (@job) {
+   $job_todo->up;
+   $job_name_maxlen = length($_->{name}) if length($_->{name}) > $job_name_maxlen;
+   }
 
 # slice of ressources for parallel job
@@ -108 +137 @@
    }
 
-my $job_num   = 0;
 my %scheduled = ();
 
@@ -125 +153 @@
         JOB:
    for my $job (@job) {
-      $job_num++;
+      my $job_name = $job->{name};
+      my $job_cmd  = $job->{cmd};
 
       # job has been already run ?
-      if (exists $state{$job_num}) {
-         if ($state{$job_num} eq 'start') {
-            print "warning: job $job_num was not clearly finished, relaunching...\n"
+      if (exists $state{$job_name}) {
+         if ($state{$job_name} eq 'start') {
+            print "warning: job $job_name was not clearly finished, relaunching...\n"
                if $verbose;
             }
-         elsif ($state{$job_num} eq 'end') {
-            delete $state{$job_num}; # free memory
+         elsif ($state{$job_name} eq 'end') {
+            delete $state{$job_name}; # free memory
             $job_todo->down;
-            print "warning: job $job_num already run\n" if $verbose;
+            print "warning: job $job_name already run\n" if $verbose;
             cede;
             next JOB;
@@ -160 +189 @@
          node_connect => $node_connect,
          ressource    => $job_ressource,
-         num          => $job_num
+         name         => $job_name
          };
 
-      my $msg = sprintf "start job %5i / %5i at %s on node %s\n",
-         $job_num, $job_pid, time, $job_ressource;
+      my $msg = sprintf "start job %${job_name_maxlen}s / %5i at %s on node %s\n",
+         $job_name, $job_pid, time, $job_ressource;
       $log_h->print($msg) if $logtrace;
       print($msg) if $verbose;
 
       my ($job_stdout, $job_stderr);
-      $job_stdout = ">  $stdout-$job_num.stdout" if $stdout ne '' and $switchio;
-      $job_stderr = "2> $stderr-$job_num.stderr" if $stderr ne '' and $switchio;
-
-      my $job_nodefile = "/tmp/oar-parexec-$ENV{LOGNAME}-$job_num";
+      $job_stdout = ">  $stdout-$job_name.stdout" if $stdout ne '' and $switchio;
+      $job_stderr = "2> $stderr-$job_name.stderr" if $stderr ne '' and $switchio;
+
+      my $job_nodefile = "/tmp/oar-parexec-$ENV{LOGNAME}-$job_name";
 
      # set job environment, run it and clean
@@ -186 +215 @@
          }
       $fh->print("cd $current_dir\n");
-      $fh->print("$job $job_stdout $job_stderr\n");
+      $fh->print("$job_cmd $job_stdout $job_stderr\n");
       $fh->print("rm -f $job_nodefile\n") if $job_np > 1;
       $fh->print("exit\n");
@@ -199 +228 @@
                         # non blocking PID test
          if (waitpid($job_pid, WNOHANG)) {
-            my $msg = sprintf "end   job %5i / %5i at %s on node %s\n",
-               $scheduled{$job_pid}->{num},
+            my $msg = sprintf "end   job %${job_name_maxlen}s / %5i at %s on node %s\n",
+               $scheduled{$job_pid}->{name},
                $job_pid, time, $scheduled{$job_pid}->{ressource};
             $log_h->print($msg) if $logtrace;
@@ -237 +266 @@
 =head1 SYNOPSIS
 
- oar-parexec --filecmd filecommand [--logtrace tracefile] [--verbose] [--jobnp integer] \
-            [--nodefile filenode] [--masterio basefileio] [--switchio] [--oarsh sssh]
+ oar-parexec --filecmd filecommand [--logtrace tracefile] [--verbose] \
+            [--jobnp integer] [--nodefile filenode] [--masterio basefileio] [--switchio] [--oarsh sssh]
+ oar-parexec --dir foldertoitemize --cmd commandtolaunch [--logtrace tracefile] [--verbose] \
+            [--jobnp integer] [--nodefile filenode] [--masterio basefileio] [--switchio] [--oarsh sssh]
  oar-parexec --help
 
@@ -249 +280 @@
 However, it can be used outside OAR.
 
-Option C<--filecmd> is the only mandatory one.
+Option C<--filecmd> or C<--dir> and C<--cmd> are the only mandatory parameters.
 
 Small job will be launch in the same folder as the master job.
@@ -274 +305 @@
 =over 12
 
-=item B<-f|--filecmd    filecommand>
+=item B<-f|--filecmd filecommand>
 
 File name which content job list.
+For the JOB_NAME definition,
+the first valid job in the list will have the number 1 and so on...
+
+=item B<-d|--dir foldertoitemize>
+
+Command C<--cmd> will be launch in all sub-folder of this master folder.
+Files in this folder will be ignored.
+Sub-folder name which begin with '.'
+or finish with '.old', '.sav', '.bak', '.no' will either be ignored...
+
+The JOB_NAME is simply the Sub-folder name.
+
+=item B<-c|--cmd commandtolaunch>
+
+Command (and argument to it) tha will be launch in all sub-folder
+parameter folfer C<--dir>
 
 =item B<-l|--logtrace tracefile>
@@ -284 +331 @@
 only job that are not mark as done will be run again.
 Be careful, job mark as running (start but not finish) will be run again.
+Tracing is base on the JOB_NAME between multiple run.
 
 This option is very usefull in case of crash
@@ -315 +363 @@
 
 Each small job will have it's own output STDOUT and STDERR
-base on master OAR job with C<JOB_NUM> inside
+base on master OAR job with C<JOB_NAME> inside
 (or base on C<basefileio> if option C<masterio>).
 Example :
 
- OAR.151524.stdout -> OAR.151524-JOB_NUM.stdout
+ OAR.151524.stdout -> OAR.151524-JOB_NAME.stdout
 
 where 151524 here is the master C<OAR_JOB_ID>
-and C<JOB_NUM> is the small job nnumber.
+and C<JOB_NAME> is the small job name.
 
 =item B<-o|-oarsh command>