source: trunk/oarutils/oar-dispatch @ 92

Last change on this file since 92 was 92, checked in by g7moreau, 11 years ago
  • Just format as usual (no tab, 3 space...)
File size: 4.3 KB
Line 
1#!/usr/bin/perl
2#
3# 2011/11/03 gabriel
4
5use strict;
6
7use Getopt::Long();
8use Pod::Usage;
9use Coro;
10use Coro::Signal;
11use Coro::Semaphore;
12#use Coro::Timer qw(sleep);
13
14my $task = 0;
15my $overload = 1.1;
16my $file = '';
17my $verbose;
18my $help;
19
20Getopt::Long::GetOptions(
21   'task=i'       => \$task,
22   'overload=f'   => \$overload,
23   'file=s'       => \$file,
24   'verbose'      => \$verbose,
25   'help'         => \$help,
26   ) || pod2usage(-verbose => 0);
27pod2usage(-verbose => 2) if $help;
28
29if ($task == 0) {
30   open(NODE_FILE, '<', "$ENV{OAR_NODE_FILE}") or die "can't open ENV{OAR_NODE_FILE}: $!";
31   $task++ while <NODE_FILE>;
32   close NODE_FILE;
33   }
34
35# job to run
36my @job = ();
37open (JOB_LIST, '<', "$file") or die "can't open $file: $!";
38while (<JOB_LIST>) {
39   chomp;
40   next if m/^#/;
41   push @job, $_ if m/^\s*oarsub/;
42   }
43close JOB_LIST;
44
45my $container_id=$ENV{OAR_JOB_ID};
46my $insert_oar_option = "-t inner=$container_id";
47
48# interactive job
49if (not $container_id > 1) {
50   $insert_oar_option = '';
51   $overload = 1;
52   }
53
54
55my $finished = new Coro::Signal;
56my $job_active = new Coro::Semaphore 0;
57my $job_todo = new Coro::Semaphore 0;
58$job_todo->up for (@job);
59
60my %scheduled = ();
61
62# asynchrone start job block
63async {
64   JOB:
65   for my $job (@job) {
66      while ($job_active->count >= $task*$overload) {
67         cede;
68         }
69      $job =~ s/^\s*oarsub//;
70      print "oarsub $insert_oar_option $job" if $verbose;
71      my $job_id = `oarsub $insert_oar_option $job|grep ^OAR_JOB_ID|cut -f 2 -d '='`;
72      chomp $job_id;
73      if ($job_id > 1) {
74         $scheduled{$job_id}++;
75         $job_active->up;
76         }
77      cede;
78      }
79   }
80
81async {
82   while () {
83      for my $job_id (keys %scheduled) {
84         my $is_finish = `oarstat -s -j $job_id`;
85         chomp $is_finish;
86         if ($is_finish =~ m/Terminated/) {
87            delete $scheduled{$job_id};
88            $job_active->down;
89            $job_todo->down;
90            }
91         cede;
92         }
93
94      $finished->send if $job_todo->count == 0;
95      cede;
96      }
97   }
98
99cede;
100   
101# all job have been done
102$finished->wait;
103
104
105__END__
106
107=head1 NAME
108
109oar-dispatch - dispatch lot of small oar job
110
111=head1 SYNOPSIS
112
113 oar-dispatch [--task integer] [--overload real] --file filecommand [--verbose]
114 oar-dispatch --help
115
116=head1 OPTIONS
117
118=over 12
119
120=item B<[-t|--task integer]>
121
122Number of task to do in parallel.
123Default to the line number of the file OAR_NODE_FILE.
124 
125=item B<[-o|--overload real]>
126
127Number of OAR job to create / number of task.
128Some job are create in advance to start whenever it's possible.
1291.1 by default.
130
131=item B<[-f|--file filecommand]>
132
133File name which content OAR job list
134
135=item B<[-v|--verbose]>
136 
137=item B<[-h|--help]>
138
139=back
140
141Input job file name content can have
142
143 - empty line
144 - comment line begin with #
145 - oarsub command without -t option
146 
147C<oar-dispatch> will add C<-t inner=container_id> in this command line,
148just after C<oarsub>.
149
150=head1 EXAMPLE
151
152Example where the file F<$HOME/test/subjob.txt> is a list of OAR script job (and can be executable but not need here).
153
154 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob1.oar
155 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob2.oar
156 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob3.oar
157 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob4.oar
158 ...
159 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob38.oar
160 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob39.oar
161 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob40.oar
162
163These jobs could be launch with
164
165 oarsub -t container -n test-container -l /core=6,walltime=00:35:00 "oar-dispatch -f ./subjob.list.txt"
166
167Total C<walltime> is defined by the formula:
168
169 total_walltime = subjob_walltime * total_subjob / core + global_delay
170
171In practise, C<oar-dispatch> take few second and each subjob run in less than it's walltime so
172
173 total_walltime < subjob_walltime * total_subjob / core
174
175If launch in interactif, C<overload> parameter is equal to 1,
176C<task> must be define
177and no inner container is add to the C<oarsub> command line.
178
179
180=head1 SEE ALSO
181
182oar-parexec, mpilauncher
183
184
185=head1 AUTHORS
186
187Written by Gabriel Moreau, Grenoble - France
188
189
190=head1 LICENSE AND COPYRIGHT
191
192GPL version 2 or later and Perl equivalent
193
194Copyright (C) 2011 Gabriel Moreau / LEGI - CNRS UMR 5519 - France
195
Note: See TracBrowser for help on using the repository browser.