source: trunk/oarutils/oar-parexec @ 38

Last change on this file since 38 was 38, checked in by g7moreau, 12 years ago
  • Add logfile and restart fonctionality
File size: 8.0 KB
Line 
1#!/usr/bin/perl
2#
3# 2011/11/27 gabriel
4
5use strict;
6
7use Getopt::Long();
8use Pod::Usage;
9use Coro;
10use Coro::Semaphore;
11use Coro::Signal;
12use Coro::Channel;
13use Coro::Handle;
14use IO::File;
15use POSIX qw( WNOHANG WEXITSTATUS );
16use Cwd qw( getcwd );
17
18my $file = '';
19my $logfile = '';
20my $verbose;
21my $job_np = 1;
22my $nodefile = $ENV{OAR_NODE_FILE} || '';
23my $masterio;
24my $switchio;
25my $help;
26my $oarsh = 'oarsh -q -T';
27
28Getopt::Long::GetOptions(
29   'file=s'     => \$file,
30   'logfile=s'  => \$logfile,
31   'verbose'    => \$verbose,
32   'help'       => \$help,
33   'oarsh=s'    => \$oarsh,
34   'jobnp=i'    => \$job_np,
35   'nodefile=s' => \$nodefile,
36   'masterio=s' => \$masterio,
37   'switchio'   => \$switchio,
38   ) || pod2usage( -verbose => 0 );
39pod2usage( -verbose => 2 ) if $help;
40pod2usage( -verbose => 2 ) if not -e $file;
41
42my %state;
43my $log_h = IO::File->new();
44if (-e $logfile) {
45        $log_h->open("< $logfile")
46         or die "can't read log file: $!";
47   while (<$log_h>) {
48                $state{$1} = 'start' if m/^start\sjob\s(\d+)/;
49                $state{$1} = 'end'   if m/^end\sjob\s(\d+)/;
50           }
51   $log_h->close();
52   }
53if ($logfile) {
54   $log_h->open(">> $logfile")
55         or die "can't open log file: $!";
56   $log_h = unblock $log_h;
57   }
58
59my @job = ();
60open( JOB_LIST, '<', "$file" ) or die "can't open $file: $!";
61while (<JOB_LIST>) {
62   chomp;
63   next if m/^#/;
64   next if m/^\s*$/;
65   push @job, $_ ;
66   }
67close JOB_LIST;
68
69my @ressources = ();
70open( NODE_FILE, '<', "$nodefile" )
71   or die "can't open $nodefile: $!";
72while (<NODE_FILE>) {
73   chomp;
74   next if m/^#/;
75   next if m/^\s*$/;
76   push @ressources, $_ ;
77   }
78close NODE_FILE;
79
80my $ressource_size = scalar(@ressources);
81die "not enought ressources jobnp $job_np > ressources $ressource_size" if $job_np > $ressource_size;
82
83my $current_dir = getcwd();
84
85my $stderr = $ENV{OAR_STDERR} || '';
86$stderr =~ s/\.stderr$//;
87$stderr = $masterio if $masterio;
88my $stdout = $ENV{OAR_STDOUT} || '';
89$stdout =~ s/\.stdout$//;
90$stdout = $masterio if $masterio;
91
92
93my $finished = new Coro::Signal;
94my $job_todo = new Coro::Semaphore 0;
95$job_todo->up for (@job);
96
97my $ressources = new Coro::Channel;
98for my $slot (1 .. int($ressource_size / $job_np)) {
99   $ressources->put( join(',', @ressources[(($slot - 1) * $job_np) .. (($slot * $job_np) - 1)] ) );
100   }
101
102
103my $job_num   = 0;
104my %scheduled = ();
105
106async {
107   for my $job (@job) {
108      $job_num++;
109                if (exists $state{$job_num} and $state{$job_num} eq 'end') {
110                        delete $state{$job_num};
111                        cede;
112                        next;
113                   }
114                if (exists $state{$job_num} and $state{$job_num} eq 'start') {
115                        print "warning: job $job_num was not finished, relaunching...\n" if $verbose;
116                   }           
117
118      my $job_ressource = $ressources->get;
119
120      my ($node_connect) = split ',', $job_ressource;
121      my $fh      = IO::File->new();
122      my $job_pid = $fh->open("| $oarsh $node_connect >/dev/null 2>&1")
123         or die "don't start subjob: $!";
124
125      $fh->autoflush;
126      $fh = unblock $fh;
127
128      $scheduled{$job_pid} = { fh => $fh, node_connect => $node_connect, ressource => $job_ressource, num => $job_num };
129
130      $log_h->printf("start job %5i at %s\n", $job_num, time) if $logfile;
131      printf "start job %5i / %5i at %s on node %s\n",
132         $job_num, $job_pid, time, $job_ressource
133         if $verbose;
134
135      my ( $job_stdout, $job_stderr );
136      $job_stdout = ">  $stdout-$job_num.stdout" if $stdout ne '' and $switchio;
137      $job_stderr = "2> $stderr-$job_num.stderr" if $stderr ne '' and $switchio;
138
139      my $job_nodefile = "/tmp/oar-parexec-$ENV{LOGNAME}-$job_num";
140
141      if ($job_np > 1) {
142         $fh->print("printf \""
143            . join('\n',split(',',$job_ressource,))
144            . "\" > $job_nodefile\n");
145         $fh->print("OAR_NODE_FILE=$job_nodefile\n");
146         $fh->print("OAR_NP=$job_np\n");
147         $fh->print("export OAR_NODE_FILE\n");
148         $fh->print("export OAR_NP\n");
149         $fh->print("unset OAR_MSG_NODEFILE\n");
150         }
151      $fh->print("cd $current_dir\n");
152      $fh->print("$job $job_stdout $job_stderr\n");
153      $fh->print("rm -f $job_nodefile\n") if $job_np > 1;
154      $fh->print("exit\n");
155      cede;
156      }
157   }
158
159async {
160   while () {
161      for my $job_pid ( keys %scheduled ) {
162         if ( waitpid( $job_pid, WNOHANG ) ) {
163            $log_h->printf("end job %5i at %s\n", $job_num, time) if $logfile;
164            printf "end   job %5i / %5i at %s on node %s\n",
165               $scheduled{$job_pid}->{num},
166               $job_pid, time,
167               $scheduled{$job_pid}->{ressource}
168               if $verbose;
169            close $scheduled{$job_pid}->{fh};
170            $ressources->put( $scheduled{$job_pid}->{ressource} );
171            $job_todo->down;
172            delete $scheduled{$job_pid};
173            }
174         cede;
175         }
176
177      $finished->send if $job_todo->count == 0;
178      cede;
179      }
180   }
181
182cede;
183
184$finished->wait;
185
186$log_h->close if $logfile;
187
188__END__
189
190=head1 NAME
191
192oar-parexec - parallel execute lot of small job
193
194=head1 SYNOPSIS
195
196 oar-parexec --file filecommand [--verbose] [--jobnp integer] [--nodefile filenode] [--masterio basefileio] [--switchio] [--oarsh sssh]
197 oar-parexec --help
198
199=head1 DESCRIPTION
200
201C<oar-parexec> execute lot of small job.in parallel inside a cluster.
202Number of parallel job at one time cannot excede core number in the node file.
203C<oar-parexec> is easier to use inside an OAR job environment
204which define automatically theses strategics parameters...
205
206Option C<--file> is the only mandatory one.
207
208Small job will be launch in the same folder as the master job.
209Two environment variable are define for each small job
210and only in case of parallel small job (option C<--jobnp> > 1).
211
212 OAR_NODE_FILE - file that list node for parallel computing
213 OAR_NP        - number of processor affected
214
215The file define by OAR_NODE_FILE is created on the node before launching
216the small job in /tmp and will be delete after...
217C<oar-parexec> is a simple script,
218OAR_NODE_FILE will not be deleted in case of crash of the master job.
219
220OAR define other variable that are equivalent to OAR_NODE_FILE:
221OAR_NODEFILE, OAR_FILE_NODES, OAR_RESOURCE_FILE...
222You can use in your script the OAR original file ressources
223by using these variable if you need it.
224 
225
226=head1 OPTIONS
227
228=over 12
229
230=item B<-f|--file       filecommand>
231
232File name which content job list.
233
234=item B<-v|--verbose>
235
236=item B<-j|--jobnp integer>
237
238Number of processor to allocated for each small job.
2391 by default.
240
241=item B<-n|--nodefile filenode>
242
243File name that list all the node to launch job.
244By defaut, it's define automatically by OAR via
245environment variable C<OAR_NODE_FILE>.
246
247For example, if you want to use 6 core on your cluster node,
248you need to put 6 times the hostname node in this file,
249one per line...
250It's a very common file in MPI process !
251
252=item B<-m|--masterio basefileio>
253
254The C<basefileio> will be use in place of environment variable
255C<OAR_STDOUT> and C<OAR_STDERR> (without extension) to build the base name of the small job standart output
256(only use when option C<swithio> is activated).
257
258=item B<-s|--switchio>
259
260Each small job will have it's own output STDOUT and STDERR
261base on master OAR job with C<JOB_NUM> inside
262(or base on C<basefileio> if option C<masterio>).
263Example :
264
265 OAR.151524.stdout -> OAR.151524-JOB_NUM.stdout
266
267where 151524 here is the master C<OAR_JOB_ID>
268and C<JOB_NUM> is the small job nnumber.
269
270=item B<-o|-oarsh command>
271
272Command use to launch a shell on a node.
273By default
274
275        oarsh -q -T
276
277=item B<-h|--help>
278
279=back
280
281
282=head1 EXAMPLE
283
284Content for the job file (option C<--file>) could have:
285
286 - empty line
287 - comment line begin with #
288 - valid shell command
289
290Example where F<$HOME/test/subjob1.sh> is a shell script (executable).
291
292 $HOME/test/subjob1.sh
293 $HOME/test/subjob2.sh
294 $HOME/test/subjob3.sh
295 $HOME/test/subjob4.sh
296 ...
297 $HOME/test/subjob38.sh
298 $HOME/test/subjob39.sh
299 $HOME/test/subjob40.sh
300
301These jobs could be launch by
302
303 oarsub -n test -l /core=6,walltime=00:35:00 "oar-parexec -f ./subjob.list.txt"
304
305
306=head1 SEE ALSO
307
308oar-dispatch, mpilauncher
309
310
311=head1 AUTHORS
312
313Written by Gabriel Moreau, Grenoble - France
314
315
316=head1 LICENSE AND COPYRIGHT
317
318GPL version 2 or later and Perl equivalent
319
320Copyright (C) 2011 Gabriel Moreau / LEGI - CNRS UMR 5519 - France
321
Note: See TracBrowser for help on using the repository browser.