source: trunk/oar/oar-dispatch @ 28

Last change on this file since 28 was 28, checked in by g7moreau, 12 years ago
  • Just change Copyright
File size: 3.8 KB
RevLine 
[9]1#!/usr/bin/perl
2#
3# 2011/11/03 gabriel
4
5use strict;
6
7use Getopt::Long();
8use Pod::Usage;
9use Coro;
[12]10use Coro::Signal;
[9]11use Coro::Semaphore;
12
13my $task = 0;
[10]14my $overload = 1.1;
[9]15my $file = '';
16my $verbose;
17my $help;
18
19Getopt::Long::GetOptions(
20        'task=i'                => \$task,
[10]21        'overload=f'    => \$overload,
[9]22        'file=s'                => \$file,
23        'verbose'       => \$verbose,
24        'help'          => \$help,
25        ) || pod2usage(-verbose => 0);
26pod2usage(-verbose => 2) if $help;
27
28if ($task == 0) {
29        open(NODE_FILE, '<', "$ENV{OAR_NODE_FILE}") or die "can't open ENV{OAR_NODE_FILE}: $!";
30        $task++ while <NODE_FILE>;
31        close NODE_FILE;
32}
33
34my @job = ();
35open (JOB_LIST, '<', "$file") or die "can't open $file: $!";
36while (<JOB_LIST>) {
37        chomp;
38        next if m/^#/;
39        push @job, $_ if m/^\s*oarsub/;
40}
41close JOB_LIST;
42
43my $container_id=$ENV{OAR_JOB_ID};
44my $insert_oar_option = "-t inner=$container_id";
45
46# interactive job
47if (not $container_id > 1) {
48        $insert_oar_option = '';
[10]49        $overload = 1;
[9]50}
51
52
53my $finished = new Coro::Signal;
54my $job_active = new Coro::Semaphore 0;
55my $job_todo = new Coro::Semaphore 0;
56$job_todo->up for (@job);
57
58my %scheduled = ();
59
60async {
61        for my $job (@job) {
[10]62                while ($job_active->count >= $task*$overload) {
[9]63                        cede;
64                }
65                $job =~ s/^\s*oarsub//;
66                print "oarsub $insert_oar_option $job" if $verbose;
67                my $job_id = `oarsub $insert_oar_option $job|grep ^OAR_JOB_ID|cut -f 2 -d '='`;
68                chomp $job_id;
69                if ($job_id > 1) {
70                        $scheduled{$job_id}++;
71                        $job_active->up;
72                }
73                cede;
74        }
75}
76
77async {
78        while () {
79                for my $job_id (keys %scheduled) {
80                        my $is_finish = `oarstat -s -j $job_id`;
81                        chomp $is_finish;
82                        if ($is_finish =~ m/Terminated/) {
83                                delete $scheduled{$job_id};
84                                $job_active->down;
85                                $job_todo->down;
86                        }
87                        cede;
88                }
89
90                $finished->send if $job_todo->count == 0;
91                cede;
92        }
93}
94
95cede;
96   
97$finished->wait;
98
99
100__END__
101
102=head1 NAME
103
104oar-dispatch - dispatch lot of small oar job
105
106=head1 SYNOPSIS
107
[10]108 oar-dispatch [--core integer] [--overload real] --file filepath [--verbose]
[9]109 oar-dispatch --help
110
111=head1 OPTIONS
112
113 --task number of task to do in parallel.
114                        default to line number of file OAR_NODE_FILE.
115 
[10]116 --overload     number of OAR job to create / number of task.
[9]117                        Some job are create in advance to start whenever it's possible.
[10]118                        1.1 by default.
[9]119
120 --file file name which content OAR job list
121
122 --verbose
123 
124 --help
125
[11]126Input job file name content can have
[9]127
128 - empty line
129 - comment line begin with #
130 - oarsub command without -t option
131 
132C<oar-dispatch> will add C<-t inner=container_id> in this command line,
133just after C<oarsub>.
134
[11]135Example where F<$HOME/test/subjob.txt> is a list of OAR script job (and can be executable but not need here).
[9]136
137 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob1.oar
138 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob2.oar
139 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob3.oar
140 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob4.oar
141 ...
142 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob38.oar
143 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob39.oar
144 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob40.oar
145
[11]146These jobs could be launch with
[9]147
148 oarsub -t container -n test-container -l /core=6,walltime=00:35:00 "oar-dispatch -f ./subjob.list.txt"
149
[11]150Total C<walltime> is defined by the formula:
[9]151
152 total_walltime = subjob_walltime * total_subjob / core + global_delay
153
[11]154In practise, C<oar-dispatch> take few second and each subjob run in less than it's walltime so
[9]155
156 total_walltime < subjob_walltime * total_subjob / core
157
[10]158If launch in interactif, C<overload> parameter is equal to 1,
[9]159C<task> must be define
160and no inner container is add to the C<oarsub> command line.
161
162
[28]163=head1 SEE ALSO
164
165oar-parexec, mpilauncher
166
167
[9]168=head1 AUTHORS
169
[28]170Written by Gabriel Moreau, Grenoble - France
171
172
173=head1 LICENSE AND COPYRIGHT
174
175GPL version 2 or later and Perl equivalent
176
177Copyright (C) 2011 Gabriel Moreau / LEGI - CNRS UMR 5519 - France
178
Note: See TracBrowser for help on using the repository browser.