Changes in / [20:30]


Ignore:
Location:
/trunk/oar
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • /trunk/oar/Makefile

    r20 r30  
    22CC=mpiCC
    33CCFLAGS=-O2
     4PODFLAG=-c 'User Contributed OAR Documentation' -r 'LEGI Trokata'
     5
    46
    57.PHONY: all clean distclean
    68
    7 all: mpilauncher
     9all: mpilauncher mpilauncher.1 oar-envsh.1 oar-dispatch.1 oar-parexec.1
    810
    911clean:
    … …  
    1921%.o: %.cpp
    2022        $(CC) $(CCFLAGS) -c $<
     23
     24%.1: %.cpp
     25        sed -re 's#^[/ ]\* ?##; s#^// ?##' $< | pod2man -n $* $(PODFLAG) > $@
     26
     27%.1: %
     28        pod2man -n $* $(PODFLAG) $< > $@
     29
     30# astyle --style=banner -s3 mpilauncher.cpp

  • /trunk/oar/mpilauncher.cpp

    r20 r30  
    1414
    1515/*
    16 These two variables sets the command to launch in each sub-directory and the arguments required. The name of the root directory of the datasets is given as an argument when starting the MPIlauncher.
    17 const string commandToLaunch("/HA/sources/begou/SOURCES/MES-PROJETS/MPILAUNCHER/appli.exe");
    18 const string("-l -info");
    19 */
    20 
    21 /*
    22 getListOfDir(string rep, vector<string> &files)
    23 
    24 Gets the list of tjhe subdirectories in the directory rep and returns them
    25 in a vector array of strings. Directories ";" and ".." are removed from the
    26 vector array.
    27 */
    28 int getListOfDir(string rep, vector<string> &files)
    29 {
    30   DIR *dir;
    31   struct dirent *ent;
    32   int count;
    33 
    34   count=0;
    35   dir = opendir (rep.c_str());
    36   if (dir != NULL) {
    37     /* search for the files within directory */
    38     while ((ent = readdir (dir)) != NULL)
    39       if (ent->d_type == DT_DIR) {
    40         if (strcmp(ent->d_name,".") * strcmp(ent->d_name,"..")) {
    41            count++;
    42            files.push_back(string(ent->d_name));
    43            }
    44         }
    45     closedir (dir);
    46     }
    47   else {
    48     cerr<<"Directory "<<rep.c_str()<<" not found"<<endl;
    49     }
     16 * These two variables sets the command to launch in each sub-directory
     17 * and the arguments required. The name of the root directory of the
     18 * datasets is given as an argument when starting the MPIlauncher. const
     19 * string
     20 * commandToLaunch("$HOME/SOURCES/MES-PROJETS/MPILAUNCHER/appli.exe");
     21 * const string("-l -info");
     22 */
     23
     24/*
     25 * getListOfDir(string rep, vector<string> &files)
     26 *
     27 * Gets the list of tjhe subdirectories in the directory rep and returns
     28 * them in a vector array of strings. Directories ";" and ".." are removed
     29 * from the vector array.
     30 */
     31int getListOfDir(string rep, vector < string > &files) {
     32   DIR *dir;
     33   struct dirent *ent;
     34   int count;
     35
     36   count = 0;
     37   dir = opendir(rep.c_str());
     38   if (dir != NULL) {
     39      /*
     40       * search for the files within directory
     41       */
     42      while ((ent = readdir(dir)) != NULL)
     43         if (ent->d_type == DT_DIR) {
     44            if (strcmp(ent->d_name, ".") *
     45                  strcmp(ent->d_name, "..")) {
     46               count++;
     47               files.push_back(string(ent->d_name));
     48               }
     49            }
     50      closedir(dir);
     51      }
     52   else {
     53      cerr << "Directory " << rep.c_str() << " not found" << endl;
     54      }
    5055   return count;
    51 }
    52 /* getListOfCommand(string rep, vector<string> &files)
    53 Gets the list of commands in the ascii file fich. One command per line (no wrap)
    54 in this first version.
    55 */
    56 int getListOfCommand(const string & fich, vector<string> &commands)
    57 {
    58   int count;
    59   string line;
    60  
    61   count=0;
    62   commands.clear();
    63 
    64   std::ifstream infile (fich.c_str(), std::ios_base::in);
    65   while (getline(infile, line, '\n'))
    66   {
    67     // remove all trailing blanks
    68     while(line.size() >0 && isspace(line[line.size() -1])) line.erase(line.size() -1);
    69    
    70     // no empty line
    71     if (line.size() > 0) {
    72        commands.push_back (line);
    73        count++;
    74        }
    75   }
    76   return count;
    77 }
    78 
    79 
    80 
    81 /*
    82 Main program
    83 */
    84 int main(int argc, char **argv)
    85 {
    86 vector<string> files = vector<string>();
    87 string rep; //root directory where to find subdirectories
    88 int rank, size, nbdir, stride;
    89 int nbcmd;
    90 /*
    91 These two variables sets the command to launch in each sub-directory and the arguments required. The name of the root directory of the datasets is given as an argument when starting the MPIlauncher.
    92 */
    93 string commandToLaunch;
    94 string arguments;
    95 string commandfile;
    96 
    97 string finalCommand; //command to execute
    98 
    99 MPI_Init (&argc, &argv);                      /* starts MPI */
    100 MPI_Comm_rank (MPI_COMM_WORLD, &rank);        /* get current process id */
    101 MPI_Comm_size (MPI_COMM_WORLD, &size);        /* get number of processes */
    102 
    103 if (argc<3){
    104    cout<<"USAGE:"<<endl;
    105    cout<<argv[0]<<" root_directory  command [arguments]"<<endl;
    106    cout<<argv[0]<<" -f  command_file"<<endl;
    107   }
    108 else {   
    109    if (strcmp(argv[1],"-f")==0) {
    110      /* processing a command file now. */
    111      commandfile=argv[2];
    112      nbcmd=getListOfCommand(commandfile, files);
    113 //     if (rank==0){
    114 //       cout<<"Command file is "<<commandfile<<endl;
    115 //       for (unsigned int i = 0;i < nbcmd;i++)
    116 //          cout<<"["<<files[i]<<"]"<<endl; 
    117 //       }
    118 
    119 
    120      //Number of commands should divide by number of cpus
    121 //     if (nbcmd%size != 0) {
    122 //        if(rank ==0)
    123 //        cerr <<nbcmd<<" command(s) to process in "<<commandfile
    124 //           <<" cannot fit on "<<size<<" processe(s)"
    125 //           <<endl<<"FAILED"<<endl;
    126 //        }
    127 //     else {
    128 //         execute the command
    129         int reste;
    130         stride=nbcmd/size;
    131         for (unsigned int i = 0;i < stride;i++) {
    132           cerr<<"Process "<<rank<<" execute "<<files[stride*rank+i] <<endl;
    133           system(files[stride*rank+i].c_str() );
    134           }
    135         //remaining command lines
    136         reste=nbcmd-stride*size;
    137         if(rank>0 && rank<=reste){
    138              cerr<<"Process "<<rank<<" execute "<<files[nbcmd-rank] <<endl;
    139              system(files[nbcmd-rank].c_str() );
    140              }
    141 //      }
     56   }
     57
     58/*
     59 * getListOfCommand(string rep, vector<string> &files) Gets the list of
     60 * commands in the ascii file fich. One command per line (no wrap) in this
     61 * first version.
     62 */
     63int getListOfCommand(const string & fich, vector < string > &commands) {
     64   int count;
     65   string line;
     66
     67   count = 0;
     68   commands.clear();
     69
     70   std::ifstream infile(fich.c_str(), std::ios_base::in);
     71   while (getline(infile, line, '\n')) {
     72      // remove all trailing blanks
     73      while (line.size() > 0 && isspace(line[line.size() - 1]))
     74         line.erase(line.size() - 1);
     75
     76      // no empty line
     77      if (line.size() > 0) {
     78         commands.push_back(line);
     79         count++;
     80         }
     81      }
     82   return count;
     83   }
     84
     85/*
     86 * Main program
     87 */
     88int main(int argc, char **argv) {
     89   vector < string > files = vector < string > ();
     90   string rep; // root folder where to find subfolders
     91   int rank, size, nbdir, stride;
     92   int nbcmd;
     93   /*
     94    * These two variables sets the command to launch in each
     95    * sub-directory and the arguments required. The name of the root
     96    * directory of the datasets is given as an argument when starting the
     97    * MPIlauncher.
     98    */
     99   string commandToLaunch;
     100   string arguments;
     101   string commandfile;
     102
     103   string finalCommand; // command to execute
     104
     105   MPI_Init(&argc, &argv);      // starts MPI
     106   MPI_Comm_rank(MPI_COMM_WORLD, &rank); // get current process id
     107   MPI_Comm_size(MPI_COMM_WORLD, &size); // get number of processes
     108
     109   if (argc < 3) {
     110      cout << "USAGE:" << endl;
     111      cout << argv[0] << " root_directory  command [arguments]" <<
     112           endl;
     113      cout << argv[0] << " -f  command_file" << endl;
    142114      }
    143115   else {
    144      /* processing a list of dir now */
    145      rep=string(argv[1]);
    146      commandToLaunch=string(argv[2]);
    147      for (int i=3; i<argc; i++) (arguments+=argv[i])+=" ";
    148        
    149      nbdir=getListOfDir(rep, files);
    150      
    151      //Number of dir should divide by number of cpus
    152      if (nbdir%size != 0) {
    153         if(rank ==0)
    154         cerr <<nbdir<<" dataset(s) to process cannot fit on "<<size<<" processes"<<endl<<"FAILED"<<endl;
    155         }
    156      else {
    157         // execute the command
    158         stride=nbdir/size;
    159         for (unsigned int i = 0;i < stride;i++) {
    160           string finalCommand("cd ");
    161           finalCommand +=rep;
    162           finalCommand +="/";
    163           finalCommand += files[stride*rank+i];
    164           finalCommand +=";";
    165           finalCommand +=commandToLaunch;
    166           finalCommand +=" ";
    167           finalCommand +=arguments;
    168           //cout<<"On "<<rank<<" execute "<<finalCommand<<endl;
    169           system(finalCommand.c_str() );
    170           }
    171         }
    172       } 
    173    }   
    174 MPI_Finalize();
    175 
    176 return 0;
    177 }
     116      if (strcmp(argv[1], "-f") == 0) {
     117         /*
     118          * processing a command file now.
     119          */
     120         commandfile = argv[2];
     121         nbcmd = getListOfCommand(commandfile, files);
     122         int reste;
     123         stride = nbcmd / size;
     124         for (unsigned int i = 0; i < stride; i++) {
     125            cerr << "Process " << rank << " execute " <<
     126                 files[stride * rank + i] << endl;
     127            system(files[stride * rank + i].c_str());
     128            }
     129         // remaining command lines
     130         reste = nbcmd - stride * size;
     131         if (rank > 0 && rank <= reste) {
     132            cerr << "Process " << rank << " execute " <<
     133                 files[nbcmd - rank] << endl;
     134            system(files[nbcmd - rank].c_str());
     135            }
     136         }
     137      else {
     138         /*
     139          * processing a list of dir now
     140          */
     141         rep = string(argv[1]);
     142         commandToLaunch = string(argv[2]);
     143         for (int i = 3; i < argc; i++)
     144            (arguments += argv[i]) += " ";
     145
     146         nbdir = getListOfDir(rep, files);
     147
     148         // Number of dir should divide by number of cpus
     149         if (nbdir % size != 0) {
     150            if (rank == 0)
     151               cerr << nbdir <<
     152                    " dataset(s) to process cannot fit on "
     153                    << size << " processes" << endl <<
     154                    "FAILED" << endl;
     155            }
     156         else {
     157            // execute the command
     158            stride = nbdir / size;
     159            for (unsigned int i = 0; i < stride; i++) {
     160               string finalCommand("cd ");
     161               finalCommand += rep;
     162               finalCommand += "/";
     163               finalCommand +=
     164                  files[stride * rank + i];
     165               finalCommand += ";";
     166               finalCommand += commandToLaunch;
     167               finalCommand += " ";
     168               finalCommand += arguments;
     169               // cout<<"On "<<rank<<" execute "<<finalCommand<<endl;
     170               system(finalCommand.c_str());
     171               }
     172            }
     173         }
     174      }
     175   MPI_Finalize();
     176
     177   return 0;
     178   }
     179
     180/*
     181 * Documentation in Perl POD format (man perlpod)
     182 *
     183 * =head1 NAME
     184 *
     185 *  mpilauncher - parallel execute lot of small job via mpi
     186 *
     187 * =head1 SYNOPSIS
     188 *
     189 *  mpilauncher root_folder command [args]
     190 *  mpilauncher -f command_file
     191 *
     192 * =head1 DESCRIPTION
     193 *
     194 * C<mpilauncher> need to be executed inside an MPI environment (mpirun),
     195 * typically a cluster... Job process are divide by the number of core and
     196 * are launched on each core via the "system" command one after other.
     197 *
     198 * There is two case: jobs are list and define in a file (option -f) or
     199 * one command is launched inside a lot of folder. In this last case, you
     200 * need to give the root folder path in which you have all your subfolder...
     201 *
     202 *
     203 * =head1 SEE ALSO
     204 *
     205 * oar-dispatch, oar-parexec
     206 *
     207 *
     208 * =head1 AUTHORS
     209 *
     210 * Written by Patrick Begou - Gabriel Moreau, Grenoble - France
     211 *
     212 *
     213 * =head1 LICENSE AND COPYRIGHT
     214 *
     215 * GPL version 2 or later
     216 *
     217 * Copyright (C) 2011 Patrick Begou / LEGI - CNRS UMR 5519 - France
     218 *
     219 * =cut
     220 *
     221 */

  • /trunk/oar/oar-dispatch

    r20 r30  
    106106=head1 SYNOPSIS
    107107
    108  oar-dispatch [--core integer] [--overload real] --file filepath [--verbose]
     108 oar-dispatch [--task integer] [--overload real] --file filepath [--verbose]
    109109 oar-dispatch --help
    110110
    111111=head1 OPTIONS
    112112
    113  --task number of task to do in parallel.
    114                         default to line number of file OAR_NODE_FILE.
     113=over 12
     114
     115=item B<[-t|--task integer]>
     116
     117Number of task to do in parallel.
     118Default to the line number of the file OAR_NODE_FILE.
    115119 
    116  --overload     number of OAR job to create / number of task.
    117                         Some job are create in advance to start whenever it's possible.
    118                         1.1 by default.
     120=item B<[-o|--overload real]>
    119121
    120  --file file name which content OAR job list
     122Number of OAR job to create / number of task.
     123Some job are create in advance to start whenever it's possible.
     1241.1 by default.
    121125
    122  --verbose
     126=item B<[-f|--file filepath]>
     127
     128File name which content OAR job list
     129
     130=item B<[-v|--verbose]>
    123131 
    124  --help
     132=item B<[-h|--help]>
     133
     134=back
    125135
    126136Input job file name content can have
    … …  
    133143just after C<oarsub>.
    134144
    135 Example where F<$HOME/test/subjob.txt> is a list of OAR script job (and can be executable but not need here).
     145=head1 EXAMPLE
     146
     147Example where the file F<$HOME/test/subjob.txt> is a list of OAR script job (and can be executable but not need here).
    136148
    137149 oarsub -n test -l /core=1,walltime=00:05:00 $HOME/test/subjob1.oar
    … …  
    161173
    162174
     175=head1 SEE ALSO
     176
     177oar-parexec, mpilauncher
     178
     179
    163180=head1 AUTHORS
    164181
    165 Gabriel Moreau (C) 2011
     182Written by Gabriel Moreau, Grenoble - France
     183
     184
     185=head1 LICENSE AND COPYRIGHT
     186
     187GPL version 2 or later and Perl equivalent
     188
     189Copyright (C) 2011 Gabriel Moreau / LEGI - CNRS UMR 5519 - France
     190

  • /trunk/oar/oar-envsh

    r20 r30  
    2828
    2929exec /opt/oar/current/bin/oarsh $HOST "$MYENV $@"
     30
     31exit
     32
     33################################################################
     34
     35Documentation in Perl POD format (man perlpod)
     36
     37=head1 NAME
     38
     39 oar-envsh - oarsh with env variable transmit
     40
     41=head1 SYNOPSIS
     42
     43 oar-envsh node command
     44
     45=head1 DESCRIPTION
     46
     47C<oar-envsh> need to be executed inside an OAR cluster.
     48It's a simple wrapper around C<oarsh> command
     49which is a wrapper around C<ssh> command!
     50
     51With C<oar-envsh>, almost all env variable are export
     52in the new shell except the following:
     53
     54 USER TERM OAR* SGE_* LS_COLORS
     55 ENV BASH_ENV HOSTNAME LOGNAME MAIL
     56 MANPATH OMPI_MCA_plm_rsh_agent PWD
     57 SHELL SSH_* SUDO_COMMAND= HOME DISPLAY
     58 SHLVL
     59
     60Alway use C<oarsh>,
     61only use C<oar-envsh> if really needed!
     62
     63
     64=head1 SEE ALSO
     65
     66oarsh
     67
     68
     69=head1 AUTHORS
     70
     71Written by :
     72
     73 Nicolas Capit, Grenoble - France
     74 Patrick Begou - Gabriel Moreau, Grenoble - France
     75
     76
     77=head1 LICENSE AND COPYRIGHT
     78
     79GPL version 2 or later
     80
     81Copyright (C) 2011 Patrick Begou / LEGI - CNRS UMR 5519 - France
     82
     83=cut

  • /trunk/oar/oar-parexec

    r20 r30  
    2929   ) || pod2usage( -verbose => 0 );
    3030pod2usage( -verbose => 2 ) if $help;
    31 pod2usage( -verbose => 2 ) if -e $file;
     31pod2usage( -verbose => 2 ) if not -e $file;
    3232
    3333my @job = ();
    … …  
    147147 --help
    148148
    149 File name content can have
     149
     150=head1 DESCRIPTION
     151
     152C<oar-parexec> need to be executed inside an OAR job environment.
     153because it need the two environment variable that OAR define by
     154default:
     155
     156 OAR_NODE_FILE path to a file which content one node by line
     157
     158 OAR_WORKDIR   dir to launch job and do a chdir inside
     159
     160Content for the job file (option C<--file>) could have:
    150161
    151162 - empty line
    … …  
    168179 oarsub -n test -l /core=6,walltime=00:35:00 "oar-parexec -f ./subjob.list.txt"
    169180
     181
     182=head1 SEE ALSO
     183
     184oar-dispatch, mpilauncher
     185
     186
    170187=head1 AUTHORS
    171188
    172 Gabriel Moreau (C) 2011
     189Written by Gabriel Moreau, Grenoble - France
    173190
     191
     192=head1 LICENSE AND COPYRIGHT
     193
     194GPL version 2 or later and Perl equivalent
     195
     196Copyright (C) 2011 Gabriel Moreau / LEGI - CNRS UMR 5519 - France
     197
Note: See TracChangeset for help on using the changeset viewer.