[9] | 1 | #!/usr/bin/perl |
---|
| 2 | # |
---|
| 3 | # 2011/11/03 gabriel |
---|
| 4 | |
---|
| 5 | use strict; |
---|
| 6 | |
---|
| 7 | use Getopt::Long(); |
---|
| 8 | use Pod::Usage; |
---|
| 9 | use Coro; |
---|
[12] | 10 | use Coro::Signal; |
---|
[9] | 11 | use Coro::Semaphore; |
---|
[93] | 12 | use Coro::Timer qw(sleep); |
---|
[9] | 13 | |
---|
| 14 | my $task = 0; |
---|
[10] | 15 | my $overload = 1.1; |
---|
[9] | 16 | my $file = ''; |
---|
| 17 | my $verbose; |
---|
| 18 | my $help; |
---|
| 19 | |
---|
| 20 | Getopt::Long::GetOptions( |
---|
[92] | 21 | 'task=i' => \$task, |
---|
| 22 | 'overload=f' => \$overload, |
---|
| 23 | 'file=s' => \$file, |
---|
| 24 | 'verbose' => \$verbose, |
---|
| 25 | 'help' => \$help, |
---|
| 26 | ) || pod2usage(-verbose => 0); |
---|
[9] | 27 | pod2usage(-verbose => 2) if $help; |
---|
| 28 | |
---|
| 29 | if ($task == 0) { |
---|
[92] | 30 | open(NODE_FILE, '<', "$ENV{OAR_NODE_FILE}") or die "can't open ENV{OAR_NODE_FILE}: $!"; |
---|
| 31 | $task++ while <NODE_FILE>; |
---|
| 32 | close NODE_FILE; |
---|
| 33 | } |
---|
[9] | 34 | |
---|
[92] | 35 | # job to run |
---|
[9] | 36 | my @job = (); |
---|
| 37 | open (JOB_LIST, '<', "$file") or die "can't open $file: $!"; |
---|
| 38 | while (<JOB_LIST>) { |
---|
[92] | 39 | chomp; |
---|
| 40 | next if m/^#/; |
---|
| 41 | push @job, $_ if m/^\s*oarsub/; |
---|
| 42 | } |
---|
[9] | 43 | close JOB_LIST; |
---|
| 44 | |
---|
| 45 | my $container_id=$ENV{OAR_JOB_ID}; |
---|
| 46 | my $insert_oar_option = "-t inner=$container_id"; |
---|
| 47 | |
---|
| 48 | # interactive job |
---|
| 49 | if (not $container_id > 1) { |
---|
[92] | 50 | $insert_oar_option = ''; |
---|
| 51 | $overload = 1; |
---|
| 52 | } |
---|
[9] | 53 | |
---|
| 54 | |
---|
| 55 | my $finished = new Coro::Signal; |
---|
| 56 | my $job_active = new Coro::Semaphore 0; |
---|
| 57 | my $job_todo = new Coro::Semaphore 0; |
---|
| 58 | $job_todo->up for (@job); |
---|
| 59 | |
---|
| 60 | my %scheduled = (); |
---|
| 61 | |
---|
[92] | 62 | # asynchrone start job block |
---|
[9] | 63 | async { |
---|
[92] | 64 | JOB: |
---|
| 65 | for my $job (@job) { |
---|
| 66 | while ($job_active->count >= $task*$overload) { |
---|
| 67 | cede; |
---|
| 68 | } |
---|
| 69 | $job =~ s/^\s*oarsub//; |
---|
| 70 | print "oarsub $insert_oar_option $job" if $verbose; |
---|
| 71 | my $job_id = `oarsub $insert_oar_option $job|grep ^OAR_JOB_ID|cut -f 2 -d '='`; |
---|
| 72 | chomp $job_id; |
---|
| 73 | if ($job_id > 1) { |
---|
| 74 | $scheduled{$job_id}++; |
---|
| 75 | $job_active->up; |
---|
| 76 | } |
---|
| 77 | cede; |
---|
[93] | 78 | |
---|
| 79 | # asynchrone guard for job end |
---|
| 80 | async { |
---|
| 81 | my $job_id = shift; |
---|
| 82 | GUARD: |
---|
| 83 | while () { |
---|
| 84 | sleep 15; # async, do not re-launch oarstat to fast |
---|
| 85 | my $is_finish = `oarstat -s -j $job_id`; |
---|
| 86 | chomp $is_finish; |
---|
| 87 | last GUARD if $is_finish =~ m/Terminated/; |
---|
| 88 | } |
---|
| 89 | delete $scheduled{$job_id}; |
---|
| 90 | $job_active->down; |
---|
| 91 | $job_todo->down; |
---|
| 92 | } $job_id; |
---|
[92] | 93 | } |
---|
[93] | 94 | }; |
---|
[9] | 95 | |
---|
| 96 | async { |
---|
[92] | 97 | while () { |
---|
[93] | 98 | # for my $job_id (keys %scheduled) { |
---|
| 99 | # my $is_finish = `oarstat -s -j $job_id`; |
---|
| 100 | # chomp $is_finish; |
---|
| 101 | # if ($is_finish =~ m/Terminated/) { |
---|
| 102 | # delete $scheduled{$job_id}; |
---|
| 103 | # $job_active->down; |
---|
| 104 | # $job_todo->down; |
---|
| 105 | # } |
---|
| 106 | # cede; |
---|
| 107 | # } |
---|
[9] | 108 | |
---|
[92] | 109 | $finished->send if $job_todo->count == 0; |
---|
| 110 | cede; |
---|
| 111 | } |
---|
[93] | 112 | }; |
---|
[9] | 113 | |
---|
| 114 | cede; |
---|
| 115 | |
---|
[92] | 116 | # all job have been done |
---|
[9] | 117 | $finished->wait; |
---|
| 118 | |
---|
| 119 | |
---|
| 120 | __END__ |
---|
| 121 | |
---|
| 122 | =head1 NAME |
---|
| 123 | |
---|
| 124 | oar-dispatch - dispatch lot of small oar job |
---|
| 125 | |
---|
| 126 | =head1 SYNOPSIS |
---|
| 127 | |
---|
[33] | 128 | oar-dispatch [--task integer] [--overload real] --file filecommand [--verbose] |
---|
[9] | 129 | oar-dispatch --help |
---|
| 130 | |
---|
| 131 | =head1 OPTIONS |
---|
| 132 | |
---|
[30] | 133 | =over 12 |
---|
| 134 | |
---|
| 135 | =item B<[-t|--task integer]> |
---|
| 136 | |
---|
| 137 | Number of task to do in parallel. |
---|
| 138 | Default to the line number of the file OAR_NODE_FILE. |
---|
[9] | 139 | |
---|
[30] | 140 | =item B<[-o|--overload real]> |
---|
[9] | 141 | |
---|
[30] | 142 | Number of OAR job to create / number of task. |
---|
| 143 | Some job are create in advance to start whenever it's possible. |
---|
| 144 | 1.1 by default. |
---|
[9] | 145 | |
---|
[33] | 146 | =item B<[-f|--file filecommand]> |
---|
[30] | 147 | |
---|
| 148 | File name which content OAR job list |
---|
| 149 | |
---|
| 150 | =item B<[-v|--verbose]> |
---|
[9] | 151 | |
---|
[30] | 152 | =item B<[-h|--help]> |
---|
[9] | 153 | |
---|
[30] | 154 | =back |
---|
| 155 | |
---|
[11] | 156 | Input job file name content can have |
---|
[9] | 157 | |
---|
| 158 | - empty line |
---|
| 159 | - comment line begin with # |
---|
| 160 | - oarsub command without -t option |
---|
| 161 | |
---|
| 162 | C<oar-dispatch> will add C<-t inner=container_id> in this command line, |
---|
| 163 | just after C<oarsub>. |
---|
| 164 | |
---|
[30] | 165 | =head1 EXAMPLE |
---|
[9] | 166 | |
---|
[30] | 167 | Example where the file F<$HOME/test/subjob.txt> is a list of OAR script job (and can be executable but not need here). |
---|
| 168 | |
---|
[9] | 169 | oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob1.oar |
---|
| 170 | oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob2.oar |
---|
| 171 | oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob3.oar |
---|
| 172 | oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob4.oar |
---|
| 173 | ... |
---|
| 174 | oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob38.oar |
---|
| 175 | oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob39.oar |
---|
| 176 | oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob40.oar |
---|
| 177 | |
---|
[11] | 178 | These jobs could be launch with |
---|
[9] | 179 | |
---|
| 180 | oarsub -t container -n test-container -l /core=6,walltime=00:35:00 "oar-dispatch -f ./subjob.list.txt" |
---|
| 181 | |
---|
[11] | 182 | Total C<walltime> is defined by the formula: |
---|
[9] | 183 | |
---|
| 184 | total_walltime = subjob_walltime * total_subjob / core + global_delay |
---|
| 185 | |
---|
[11] | 186 | In practise, C<oar-dispatch> take few second and each subjob run in less than it's walltime so |
---|
[9] | 187 | |
---|
| 188 | total_walltime < subjob_walltime * total_subjob / core |
---|
| 189 | |
---|
[10] | 190 | If launch in interactif, C<overload> parameter is equal to 1, |
---|
[9] | 191 | C<task> must be define |
---|
| 192 | and no inner container is add to the C<oarsub> command line. |
---|
| 193 | |
---|
| 194 | |
---|
[28] | 195 | =head1 SEE ALSO |
---|
| 196 | |
---|
| 197 | oar-parexec, mpilauncher |
---|
| 198 | |
---|
| 199 | |
---|
[9] | 200 | =head1 AUTHORS |
---|
| 201 | |
---|
[28] | 202 | Written by Gabriel Moreau, Grenoble - France |
---|
| 203 | |
---|
| 204 | |
---|
| 205 | =head1 LICENSE AND COPYRIGHT |
---|
| 206 | |
---|
| 207 | GPL version 2 or later and Perl equivalent |
---|
| 208 | |
---|
| 209 | Copyright (C) 2011 Gabriel Moreau / LEGI - CNRS UMR 5519 - France |
---|
| 210 | |
---|