source: trunk/oarutils/oar-dispatch @ 49

Last change on this file since 49 was 33, checked in by g7moreau, 9 years ago
  • Small change in doc
File size: 4.0 KB
Line 
1#!/usr/bin/perl
2#
3# 2011/11/03 gabriel
4
5use strict;
6
7use Getopt::Long();
8use Pod::Usage;
9use Coro;
10use Coro::Signal;
11use Coro::Semaphore;
12
13my $task = 0;
14my $overload = 1.1;
15my $file = '';
16my $verbose;
17my $help;
18
19Getopt::Long::GetOptions(
20        'task=i'                => \$task,
21        'overload=f'    => \$overload,
22        'file=s'                => \$file,
23        'verbose'       => \$verbose,
24        'help'          => \$help,
25        ) || pod2usage(-verbose => 0);
26pod2usage(-verbose => 2) if $help;
27
28if ($task == 0) {
29        open(NODE_FILE, '<', "$ENV{OAR_NODE_FILE}") or die "can't open ENV{OAR_NODE_FILE}: $!";
30        $task++ while <NODE_FILE>;
31        close NODE_FILE;
32}
33
34my @job = ();
35open (JOB_LIST, '<', "$file") or die "can't open $file: $!";
36while (<JOB_LIST>) {
37        chomp;
38        next if m/^#/;
39        push @job, $_ if m/^\s*oarsub/;
40}
41close JOB_LIST;
42
43my $container_id=$ENV{OAR_JOB_ID};
44my $insert_oar_option = "-t inner=$container_id";
45
46# interactive job
47if (not $container_id > 1) {
48        $insert_oar_option = '';
49        $overload = 1;
50}
51
52
53my $finished = new Coro::Signal;
54my $job_active = new Coro::Semaphore 0;
55my $job_todo = new Coro::Semaphore 0;
56$job_todo->up for (@job);
57
58my %scheduled = ();
59
60async {
61        for my $job (@job) {
62                while ($job_active->count >= $task*$overload) {
63                        cede;
64                }
65                $job =~ s/^\s*oarsub//;
66                print "oarsub $insert_oar_option $job" if $verbose;
67                my $job_id = `oarsub $insert_oar_option $job|grep ^OAR_JOB_ID|cut -f 2 -d '='`;
68                chomp $job_id;
69                if ($job_id > 1) {
70                        $scheduled{$job_id}++;
71                        $job_active->up;
72                }
73                cede;
74        }
75}
76
77async {
78        while () {
79                for my $job_id (keys %scheduled) {
80                        my $is_finish = `oarstat -s -j $job_id`;
81                        chomp $is_finish;
82                        if ($is_finish =~ m/Terminated/) {
83                                delete $scheduled{$job_id};
84                                $job_active->down;
85                                $job_todo->down;
86                        }
87                        cede;
88                }
89
90                $finished->send if $job_todo->count == 0;
91                cede;
92        }
93}
94
95cede;
96   
97$finished->wait;
98
99
100__END__
101
102=head1 NAME
103
104oar-dispatch - dispatch lot of small oar job
105
106=head1 SYNOPSIS
107
108 oar-dispatch [--task integer] [--overload real] --file filecommand [--verbose]
109 oar-dispatch --help
110
111=head1 OPTIONS
112
113=over 12
114
115=item B<[-t|--task integer]>
116
117Number of task to do in parallel.
118Default to the line number of the file OAR_NODE_FILE.
119 
120=item B<[-o|--overload real]>
121
122Number of OAR job to create / number of task.
123Some job are create in advance to start whenever it's possible.
1241.1 by default.
125
126=item B<[-f|--file filecommand]>
127
128File name which content OAR job list
129
130=item B<[-v|--verbose]>
131 
132=item B<[-h|--help]>
133
134=back
135
136Input job file name content can have
137
138 - empty line
139 - comment line begin with #
140 - oarsub command without -t option
141 
142C<oar-dispatch> will add C<-t inner=container_id> in this command line,
143just after C<oarsub>.
144
145=head1 EXAMPLE
146
147Example where the file F<$HOME/test/subjob.txt> is a list of OAR script job (and can be executable but not need here).
148
149 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob1.oar
150 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob2.oar
151 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob3.oar
152 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob4.oar
153 ...
154 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob38.oar
155 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob39.oar
156 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob40.oar
157
158These jobs could be launch with
159
160 oarsub -t container -n test-container -l /core=6,walltime=00:35:00 "oar-dispatch -f ./subjob.list.txt"
161
162Total C<walltime> is defined by the formula:
163
164 total_walltime = subjob_walltime * total_subjob / core + global_delay
165
166In practise, C<oar-dispatch> take few second and each subjob run in less than it's walltime so
167
168 total_walltime < subjob_walltime * total_subjob / core
169
170If launch in interactif, C<overload> parameter is equal to 1,
171C<task> must be define
172and no inner container is add to the C<oarsub> command line.
173
174
175=head1 SEE ALSO
176
177oar-parexec, mpilauncher
178
179
180=head1 AUTHORS
181
182Written by Gabriel Moreau, Grenoble - France
183
184
185=head1 LICENSE AND COPYRIGHT
186
187GPL version 2 or later and Perl equivalent
188
189Copyright (C) 2011 Gabriel Moreau / LEGI - CNRS UMR 5519 - France
190
Note: See TracBrowser for help on using the repository browser.