Changeset 918


Ignore:
Timestamp:
Jun 26, 2015, 9:20:48 AM (9 years ago)
Author:
sommeria
Message:

job launching procedure modified in series

Location:
trunk/src
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • trunk/src/series.m

    r917 r918  
    14801480end
    14811481
    1482 %% Get RunTime code from the file PARAM.xml (needed to run compiled functions)
     1482%% Get  PARAM.xml (not used at this stage)
    14831483errormsg='';%default error message
    14841484xmlfile=fullfile(path_series,'PARAM.xml');
     
    17611761        '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n'])
    17621762    RunMode = 'python';
    1763     t = struct2xml(Param);
    1764     t = set(t, 1, 'name', 'Series');
    1765     filexml = fullfile_uvmat(DirXml, '', Param.InputTable{1,3}, '.xml', OutputNomType,...
    1766          Param.IndexRange.first_i, Param.IndexRange.last_i, first_j, last_j);
    1767     save(t, filexml);
    17681763end
    17691764
    17701765
    17711766%% direct processing on the current Matlab session
    1772 if strcmp (RunMode,'local')
    1773     for iprocess=1:NbProcess
    1774         if ~strcmp(get(handles.RUN,'BusyAction'),'queue')% allow for STOP action
    1775             disp('program stopped by user')
    1776             return
    1777         end
    1778         if isempty(Param.IndexRange.NbSlice)
    1779             %Param.IndexRange.first_i=first_i+(iprocess-1)*BlockLength*incr_i;
    1780             Param.IndexRange.first_i=ref_i(1+(iprocess-1)*BlockLength);
    1781             if Param.IndexRange.first_i>last_i
    1782                 break
    1783             end
    1784             Param.IndexRange.last_i=min(ref_i(iprocess*BlockLength),last_i);
    1785             %Param.IndexRange.last_i=min(first_i+(iprocess)*BlockLength*incr_i-1,last_i);
    1786         else %multislices (then incr_i is not empty)
    1787             Param.IndexRange.first_i= first_i+incr_i*(iprocess-1);
    1788             Param.IndexRange.incr_i=incr_i*Param.IndexRange.NbSlice;
    1789         end
    1790         if isfield(Param,'OutputSubDir')
    1791             t=struct2xml(Param);
    1792             t=set(t,1,'name','Series');
    1793             filexml=fullfile_uvmat(DirXml,'',Param.InputTable{1,3},'.xml',OutputNomType,...
    1794                 Param.IndexRange.first_i,Param.IndexRange.last_i,first_j,last_j);
    1795             save(t,filexml);
    1796         end
     1767% if strcmp (RunMode,'local')
     1768for iprocess=1:NbProcess
     1769    if ~strcmp(get(handles.RUN,'BusyAction'),'queue')% allow for STOP action
     1770        disp('program stopped by user')
     1771        return
     1772    end
     1773    if isempty(Param.IndexRange.NbSlice)
     1774        Param.IndexRange.first_i=first_i+(iprocess-1)*BlockLength*incr_i;
     1775%         Param.IndexRange.first_i=ref_i(1+(iprocess-1)*BlockLength);
     1776        if Param.IndexRange.first_i>last_i
     1777            NbProcess=iprocess-1;% leave the loop, we are at the end of the calculation
     1778            break
     1779        end
     1780        %Param.IndexRange.last_i=min(ref_i(iprocess*BlockLength),last_i);
     1781        %Param.IndexRange.last_i=min(first_i+(iprocess)*BlockLength*incr_i-1,last_i);
     1782        Param.IndexRange.last_i=min(last_i,first_i+(iprocess)*BlockLength*incr_i-1);
     1783    else %multislices (then incr_i is not empty)
     1784%         Param.IndexRange.first_i= first_i+incr_i*(iprocess-1);
     1785%         Param.IndexRange.incr_i=incr_i*Param.IndexRange.NbSlice;
     1786        Param.IndexRange.first_i= first_i+iprocess-1;
     1787        Param.IndexRange.incr_i=incr_i*Param.IndexRange.NbSlice;
     1788    end
     1789    for ilist=1:size(Param.InputTable,1)
     1790        Param.InputTable{ilist,1}=regexprep(Param.InputTable{ilist,1},'\','/');%correct path name for PCWIN system
     1791    end
     1792   
     1793    %         if isempty(Param.IndexRange.NbSlice)% process by blocks of i index
     1794    %             Param.IndexRange.first_i=first_i+(iprocess-1)*BlockLength*incr_i;
     1795    %             if Param.IndexRange.first_i>last_i
     1796    %                 NbProcess=iprocess-1;
     1797    %                 break% leave the loop, we are at the end of the calculation
     1798    %             end
     1799    %             Param.IndexRange.last_i=min(last_i,first_i+(iprocess)*BlockLength*incr_i-1);
     1800    %         else% process by slices of i index if NbSlice is defined, computation in a single process if NbSlice =1
     1801    %             Param.IndexRange.first_i= first_i+iprocess-1;
     1802    %             Param.IndexRange.incr_i=incr_i*Param.IndexRange.NbSlice;
     1803    %         end
     1804   
     1805   
     1806    if isfield(Param,'OutputSubDir')
     1807        t=struct2xml(Param);
     1808        t=set(t,1,'name','Series');
     1809        extxml{iprocess}=fullfile_uvmat('','',Param.InputTable{1,3},'.xml',OutputNomType,...
     1810            Param.IndexRange.first_i,Param.IndexRange.last_i,first_j,last_j);
     1811        filexml{iprocess}=fullfile(OutputDir,'0_XML',extxml{iprocess})
     1812        save(t, filexml{iprocess});% save the xml file containing the processing parameters
     1813    end
     1814    if strcmp (RunMode,'local')
    17971815        switch ActionExt
    17981816            case '.m'
    1799                 h_fun(Param);
     1817                h_fun(Param);% direct launching
    18001818            case '.sh'
    18011819                switch computer
     
    18081826        end
    18091827    end
    1810 elseif ~strcmp(RunMode,'python')
     1828end
     1829if ~strcmp (RunMode,'local') && ~strcmp(RunMode,'python')
    18111830    %% processing on a different session of the same computer (background) or cluster, create executable files
    18121831    batch_file_list=cell(NbProcess,1);% initiate the list of executable files
     
    18351854        end
    18361855    end
     1856    %create the executable file
     1857    file_exe_global=fullfile_uvmat('','',Param.InputTable{1,3},ExeExt,OutputNomType,...
     1858        first_i,last_i,first_j,last_j);
     1859    file_exe_global=fullfile(OutputDir,'0_EXE',file_exe_global);
     1860    filelog_global=fullfile_uvmat('','',Param.InputTable{1,3},'.log',OutputNomType,...
     1861        first_i,last_i,first_j,last_j);
     1862    filelog_global=fullfile(OutputDir,'0_LOG',filelog_global);
     1863   
     1864   
    18371865    for iprocess=1:NbProcess
    1838         if isempty(Param.IndexRange.NbSlice)% process by blocks of i index
    1839             Param.IndexRange.first_i=first_i+(iprocess-1)*BlockLength*incr_i;
    1840             if Param.IndexRange.first_i>last_i
    1841                 NbProcess=iprocess-1;
    1842                 break% leave the loop, we are at the end of the calculation
    1843             end
    1844             Param.IndexRange.last_i=min(last_i,first_i+(iprocess)*BlockLength*incr_i-1);
    1845         else% process by slices of i index if NbSlice is defined, computation in a single process if NbSlice =1
    1846             Param.IndexRange.first_i= first_i+iprocess-1;
    1847             Param.IndexRange.incr_i=incr_i*Param.IndexRange.NbSlice;
    1848         end
    1849         for ilist=1:size(Param.InputTable,1)
    1850             Param.InputTable{ilist,1}=regexprep(Param.InputTable{ilist,1},'\','/');%correct path name for PCWIN system
    1851         end
    1852         % create, fill and save the xml parameter file
    1853         t=struct2xml(Param);
    1854         t=set(t,1,'name','Series');
    1855         filexml{iprocess}=fullfile_uvmat(DirXml,'',Param.InputTable{1,3},'.xml',OutputNomType,...
    1856             Param.IndexRange.first_i,Param.IndexRange.last_i,first_j,last_j);
    1857         save(t,filexml{iprocess});% save the parameter file
    18581866       
     1867        %create the executable file
     1868       
     1869        batch_file_list{iprocess}=fullfile(OutputDir,'0_EXE',regexprep(extxml{iprocess},'.xml$',ExeExt));
     1870%         [fid,message]=fopen(batch_file_list{iprocess},'w');% create the executable file
     1871%         if isequal(fid,-1)
     1872%             errormsg=['creation of ' batch_file_list{iprocess} ':' message];
     1873%             return
     1874%         end
    18591875       
    18601876        % set the log file name
    1861         filelog{iprocess}=fullfile_uvmat(DirLog,'',Param.InputTable{1,3},'.log',OutputNomType,...
    1862             Param.IndexRange.first_i,Param.IndexRange.last_i,first_j,last_j);
     1877        filelog{iprocess}=fullfile(OutputDir,'0_LOG',regexprep(extxml{iprocess},'.xml$','.log'));
     1878       
    18631879    end
    18641880end
    18651881
    18661882%% launch the executable files for background or cluster processing
     1883
    18671884switch RunMode
     1885   
    18681886    case 'background'
    1869         % create the executable file
    1870         filebat=fullfile_uvmat(DirExe,'',Param.InputTable{1,3},ExeExt,OutputNomType,...
    1871             first_i,last_i,first_j,last_j);
    1872         %batch_file_list{iprocess}=filebat;
    1873         [fid,message]=fopen(filebat,'w');% create the executable file
     1887        [fid,message]=fopen(file_exe_global,'w');
    18741888        if isequal(fid,-1)
    1875             errormsg=['creation of .bat file: ' message];
     1889            errormsg=['creation of ' file_exe_global ':' message];
    18761890            return
    18771891        end
    1878         filelog=fullfile_uvmat(DirLog,'',Param.InputTable{1,3},'.log',OutputNomType,...
    1879            first_i,last_i,first_j,last_j);
    18801892        switch ActionExt
    18811893            case '.m'% Matlab function
     
    18851897                            '#!/bin/bash \n'...
    18861898                            '. /etc/sysprofile \n'...
    1887                             'matlab -nodisplay -nosplash -nojvm -logfile ''' filelog ''' <<END_MATLAB \n'...
     1899                            'matlab -nodisplay -nosplash -nojvm -logfile ''' filelog_global ''' <<END_MATLAB \n'...
    18881900                            'addpath(''' path_series '''); \n'...
    18891901                            'addpath(''' Param.Action.ActionPath '''); \n'];
     
    18941906                        fprintf(fid,cmd);%fill the executable file with the  char string cmd
    18951907                        fclose(fid);% close the executable file
    1896                         system(['chmod +x ' filebat]);% set the file to executable
     1908                        system(['chmod +x ' file_exe_global]);% set the file to executable
    18971909                    case {'PCWIN','PCWIN64'}
    18981910                        cmd=['matlab -automation -logfile ' regexprep(filelog{iprocess},'\\','\\\\')...
     
    19001912                            'addpath(''' regexprep(Param.Action.ActionPath,'\\','\\\\') ''');'];
    19011913                        for iprocess=1:NbProcess
    1902                             cmd=[cmd '' Param.Action.ActionName  '( ''' regexprep(filexml{iprocess},'\\','\\\\') ''');']   
     1914                            cmd=[cmd '' Param.Action.ActionName  '( ''' regexprep(filexml{iprocess},'\\','\\\\') ''');']
    19031915                        end
    19041916                        cmd=[cmd ';exit"'];
     
    19061918                        fclose(fid);% close the executable file
    19071919                end
    1908                 system([filebat ' &'])% directly execute the command file
     1920                system([file_exe_global ' &'])% directly execute the command file
    19091921            case '.sh' % compiled Matlab function
    1910                 switch computer
    1911                     case {'GLNX86','GLNXA64','MACI64'}
    1912                         cmd=['#!/bin/bash \n '...
    1913                             '#$ -cwd \n '...
    1914                             'sleep $(( $RANDOM / 3000 )) \n '...
    1915                             'hostname && date \n '...
    1916                             'umask 002 \n']
    1917                         for iprocess=1:NbProcess
    1918                             cmd=[cmd fullfile(ActionPath,[ActionName '.sh']) ' ' RunTime ' ' filexml{iprocess}];%allow writting access to created files for user group
    1919                         end
    1920                         fprintf(fid,cmd);%fill the executable file with the  char string cmd
    1921                         fclose(fid);% close the executable file
    1922                         system(['chmod +x ' filebat]);% set the file to executable
    1923                         system([filebat ' &'])% directly execute the command file
    1924                     case {'PCWIN','PCWIN64'}
    1925                         msgbox_uvmat('ERROR','option for compiled Matlab functions not implemented for Windows system')
    1926                         return
     1922                for iprocess=1:NbProcess
     1923                    switch computer
     1924                        case {'GLNX86','GLNXA64','MACI64'}
     1925                            [fid,message]=fopen(batch_file_list{iprocess},'w');% create the executable file
     1926                            if isequal(fid,-1)
     1927                                errormsg=['creation of .bat file: ' message];
     1928                                return
     1929                            end
     1930                            cmd=['#!/bin/bash \n '...
     1931                                '#$ -cwd \n '...
     1932                                'hostname && date \n '...
     1933                                'umask 002 \n'...
     1934                                fullfile(ActionPath,[ActionName '.sh']) ' ' RunTime ' ' filexml];%allow writting access to created files for user group
     1935                            fprintf(fid,cmd);%fill the executable file with the  char string cmd
     1936                            fclose(fid);% close the executable file
     1937                            system(['chmod +x ' batch_file_list{iprocess}]);% set the file to executable
     1938                            system([batch_file_list{iprocess} ' &'])% directly execute the command file
     1939                        case {'PCWIN','PCWIN64'}
     1940                            msgbox_uvmat('ERROR','option for compiled Matlab functions not implemented for Windows system')
     1941                            return
     1942                    end
    19271943                end
    19281944                msgbox_uvmat('CONFIRMATION',[ActionName ' launched in background: press STATUS to see results'])
    1929             case 'cluster_oar' % option 'oar-parexec' used
    1930                 %create subdirectory for oar command and global log files
    1931                 DirOAR=fullfile(OutputDir,'0_OAR');
    1932                 if exist(DirOAR,'dir')% delete the content of the dir 0_LOG to allow new input
    1933                     curdir=pwd;
    1934                     cd(DirOAR)
    1935                     delete('*')
    1936                     cd(curdir)
    1937                 else
    1938                     [tild,msg1]=mkdir(DirOAR);
    1939                     if ~strcmp(msg1,'')
    1940                         errormsg=['cannot create ' DirOAR ': ' msg1];%error message for directory creation
    1941                         return
    1942                     end
    1943                 end
    1944                 filename_joblist=fullfile(DirOAR,'job_list.txt');%create name of the global executable file
    1945                 filename_log=fullfile(DirLog,'job_list.stdout');%file for output messages of the master oar process
    1946                 filename_errors=fullfile(DirLog,'job_list.stderr');%file for error messages of the master oar process
    1947                
    1948                 fid=fopen(filename_joblist,'w');
    1949                 for p=1:length(batch_file_list)
    1950                     fprintf(fid,[batch_file_list{p} '\n']);% list of exe files
    1951                 end
    1952                 fclose(fid);
    1953                 system(['chmod +x ' filename_joblist]);% set the file to executable
    1954                
    1955                 % the command job_list.txt contains the list of NbProcess independent individual jobs
    1956                 % in which the total calculation has been split. Those are written as executable files .sh in the folder /O_EXE.
    1957                 %  These individual jobs are grouped by the system as oar jobs on the NbCore processors.
    1958                 %  For each processor, the oar job must stop after the walltime which has been set, which is limited to 24 h.
    1959                 %  However, the oar job is automatically restarted (option 'idempotent') provided the individual jobs are
    1960                 % shorter than the wall time: in the time interval 'checkpoint' (WallTimeOneJob) before the end of the allowed duration,
    1961                 %  the oar job restarts when an individual job ends.
    1962                 JobTime=CPUTime*BlockLength*nbfield_j;% estimated time for one individual job (in minutes)
    1963                 % wall time (in hours ) for each oar job, allowing 10 individual jobs, but limited to 23 h:
    1964                 WallTimeTotal=min(23,4*JobTime/60);
    1965                 %disp(['WallTimeTotal: ' num2str(WallTimeTotal) ' hours'])
    1966                 % estimated time of an individual job (in min), with a margin of error
    1967                 WallTimeOneJob=min(4*JobTime+10,WallTimeTotal*60/2);% estimated max time of an individual job for checkpoint
    1968                 disp(['WallTimeOneJob: ' num2str(WallTimeOneJob) ' minutes'])
    1969                 oar_command=['oarsub -n UVmat_' ActionName ' '...
    1970                     '-t idempotent --checkpoint ' num2str(WallTimeOneJob*60) ' '...
    1971                     '-l /core=' num2str(NbCore) ','...
    1972                     'walltime=' datestr(WallTimeTotal/24,13) ' '...
    1973                     '-E ' filename_errors ' '...
    1974                     '-O ' filename_log ' '...
    1975                     extra_oar ' '...
    1976                     '"oar-parexec -s -f ' filename_joblist ' '...
    1977                     '-l ' filename_joblist '.log"'];
    1978                 fprintf(oar_command);% display  system command on the Matlab command window
    1979                 [status,result]=system(oar_command)% execute system command and show the result (ID number of the launched job) on the Matlab command window
    1980                 filename_oarcommand=fullfile(DirOAR,'0_oar_command');% keep track of the command in file '0-OAR/0_oar_command'
    1981                 fid=fopen(filename_oarcommand,'w');
    1982                 fprintf(fid,oar_command); % store the command
    1983                 fprintf(fid,result);% store the result (job ID number)
    1984                 fclose(fid);
    1985                
    1986                 msgbox_uvmat('CONFIRMATION',[ActionName ' launched as  ' num2str(NbProcess) ' processes in cluster: press STATUS to see results'])
    1987             case 'cluster_pbs' % for LMFA Kepler machine
    1988                 %create subdirectory for pbs command and log files
    1989                 DirPBS=fullfile(OutputDir,'0_PBS'); %todo : common name OAR/PBS
    1990                 if exist(DirPBS,'dir')% delete the content of the dir 0_LOG to allow new input
    1991                     curdir=pwd;
    1992                     cd(DirPBS)
    1993                     delete('*')
    1994                     cd(curdir)
    1995                 else
    1996                     [tild,msg1]=mkdir(DirPBS);
    1997                     if ~strcmp(msg1,'')
    1998                         errormsg=['cannot create ' DirPBS ': ' msg1];%error message for directory creation
    1999                         return
    2000                     end
    2001                 end
    2002                 max_walltime=3600*20; % 20h max total calculation (cannot exceed 24 h)
    2003                 walltime_onejob=1800; % seconds, max estimated time for asingle file index value
    2004                 filename_joblist=fullfile(DirPBS,'job_list.txt');%create name of the global executable file
    2005                 fid=fopen(filename_joblist,'w');
    2006                 for p=1:length(batch_file_list)
    2007                     fprintf(fid,[batch_file_list{p} '\n']);% list of exe files
    2008                 end
    2009                 fclose(fid);
    2010                 system(['chmod +x ' filename_joblist]);% set the file to executable
    2011                 pbs_command=['qstat -n CIVX '...
    2012                     '-t idempotent --checkpoint ' num2str(walltime_onejob+60) ' '...
    2013                     '-l /core=' num2str(NbCore) ','...
    2014                     'walltime=' datestr(min(1.05*walltime_onejob/86400*max(NbProcess*BlockLength*nbfield_j,NbCore)/NbCore,max_walltime/86400),13) ' '...
    2015                     '-E ' regexprep(filename_joblist,'\.txt\>','.stderr') ' '...
    2016                     '-O ' regexprep(filename_joblist,'\.txt\>','.log') ' '...
    2017                     extra_oar ' '...
    2018                     '"oar-parexec -s -f ' filename_joblist ' '...
    2019                     '-l ' filename_joblist '.log"'];
    2020                 filename_oarcommand=fullfile(DirPBS,'pbs_command');
    2021                 fid=fopen(filename_oarcommand,'w');
    2022                 fprintf(fid,pbs_command);
    2023                 fclose(fid);
    2024                 fprintf(pbs_command);% display in command line
    2025                 %system(pbs_command);
    2026                 msgbox_uvmat('CONFIRMATION',[ActionName ' command ready to be launched in cluster'])
    2027             case 'python'
    2028                 command = [
    2029                     'LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | pyp "p.split('':'') | [s for s in p if ''matlab'' not in s] | '':''.join(p)") ' ...
    2030                     'python -m fluiddyn.postproc.uvmat ' filexml{iprocess}];
    2031                 % fprintf(['command:\n' command '\n\n'])
    2032                 system(command, '-echo');
    2033         end
    2034 end
     1945        end
     1946       
     1947    case 'cluster_oar' % option 'oar-parexec' used
     1948        %create subdirectory for oar commands
     1949        for iprocess=1:NbProcess           
     1950            [fid,message]=fopen(batch_file_list{iprocess},'w');% create the executable file
     1951            if isequal(fid,-1)
     1952                errormsg=['creation of .bat file: ' message];
     1953                return
     1954            end
     1955            if  strcmp(ActionExt,'.sh')
     1956            cmd=['#!/bin/bash \n '...
     1957                '#$ -cwd \n '...
     1958                'hostname && date \n '...
     1959                'umask 002 \n'...
     1960                fullfile(ActionPath,[ActionName '.sh']) ' ' RunTime ' ' filexml{iprocess}];%allow writting access to created files for user group
     1961            else
     1962                cmd=[...
     1963                            '#!/bin/bash \n'...
     1964                            '. /etc/sysprofile \n'...
     1965                            'matlab -nodisplay -nosplash -nojvm -logfile ''' filelog{iprocess} ''' <<END_MATLAB \n'...
     1966                            'addpath(''' path_series '''); \n'...
     1967                            'addpath(''' Param.Action.ActionPath '''); \n'...
     1968                            '' Param.Action.ActionName  '( ''' filexml{iprocess} '''); \n'...
     1969                            'exit \n'...
     1970                            'END_MATLAB \n'];
     1971            end
     1972            fprintf(fid,cmd);%fill the executable file with the  char string cmd
     1973            fclose(fid);% close the executable file
     1974            system(['chmod +x ' batch_file_list{iprocess}]);% set the file to executable
     1975        end
     1976        DirOAR=fullfile(OutputDir,'0_OAR');
     1977        if exist(DirOAR,'dir')% delete the content of the dir 0_LOG to allow new input
     1978            curdir=pwd;
     1979            cd(DirOAR)
     1980            delete('*')
     1981            cd(curdir)
     1982        else
     1983            [tild,msg1]=mkdir(DirOAR);
     1984            if ~strcmp(msg1,'')
     1985                errormsg=['cannot create ' DirOAR ': ' msg1];%error message for directory creation
     1986                return
     1987            end
     1988        end
     1989        % create file containing the list of jobs
     1990        filename_joblist=fullfile(DirOAR,'job_list.txt');% name of the file contqining the list of executables
     1991        fid=fopen(filename_joblist,'w');%open it for writting
     1992        for iprocess=1:length(batch_file_list)
     1993            fprintf(fid,[batch_file_list{iprocess} '\n']);% write list of exe files
     1994        end
     1995        fclose(fid);
     1996        system(['chmod +x ' filename_joblist]);% set the file to executable
     1997       
     1998        filename_log=fullfile(DirLog,'job_list.stdout');%file for output messages of the master oar process
     1999        filename_errors=fullfile(DirLog,'job_list.stderr');%file for error messages of the master oar process
     2000        % the command job_list.txt contains the list of NbProcess independent individual jobs
     2001        % in which the total calculation has been split. Those are written as executable files .sh in the folder /O_EXE.
     2002        %  These individual jobs are grouped by the system as oar jobs on the NbCore processors.
     2003        %  For each processor, the oar job must stop after the walltime which has been set, which is limited to 24 h.
     2004        %  However, the oar job is automatically restarted (option 'idempotent') provided the individual jobs are
     2005        % shorter than the wall time: in the time interval 'checkpoint' (WallTimeOneJob) before the end of the allowed duration,
     2006        %  the oar job restarts when an individual job ends.
     2007        JobTime=CPUTime*BlockLength*nbfield_j;% estimated time for one individual job (in minutes)
     2008        % wall time (in hours ) for each oar job, allowing 10 individual jobs, but limited to 23 h:
     2009        WallTimeTotal=min(23,4*JobTime/60);
     2010        %disp(['WallTimeTotal: ' num2str(WallTimeTotal) ' hours'])
     2011        % estimated time of an individual job (in min), with a margin of error
     2012        WallTimeOneJob=min(4*JobTime+10,WallTimeTotal*60/2);% estimated max time of an individual job for checkpoint
     2013        disp(['WallTimeOneJob: ' num2str(WallTimeOneJob) ' minutes'])
     2014        oar_command=['oarsub -n UVmat_' ActionName ' '...
     2015            '-t idempotent --checkpoint ' num2str(WallTimeOneJob*60) ' '...
     2016            '-l /core=' num2str(NbCore) ','...
     2017            'walltime=' datestr(WallTimeTotal/24,13) ' '...
     2018            '-E ' filename_errors ' '...
     2019            '-O ' filename_log ' '...
     2020            extra_oar ' '...
     2021            '"oar-parexec -s -f ' filename_joblist ' '...
     2022            '-l ' filename_joblist '.log"'];
     2023        fprintf(oar_command);% display  system command on the Matlab command window
     2024        [status,result]=system(oar_command)% execute system command and show the result (ID number of the launched job) on the Matlab command window
     2025        filename_oarcommand=fullfile(DirOAR,'0_oar_command');% keep track of the command in file '0-OAR/0_oar_command'
     2026        fid=fopen(filename_oarcommand,'w');
     2027        fprintf(fid,oar_command); % store the command
     2028        fprintf(fid,result);% store the result (job ID number)
     2029        fclose(fid);
     2030        msgbox_uvmat('CONFIRMATION',[ActionName ' launched as  ' num2str(NbProcess) ' processes in cluster: press STATUS to see results'])
     2031       
     2032    case 'cluster_pbs' % for LMFA Kepler machine
     2033        %create subdirectory for pbs command and log files
     2034        DirPBS=fullfile(OutputDir,'0_PBS'); %todo : common name OAR/PBS
     2035        if exist(DirPBS,'dir')% delete the content of the dir 0_LOG to allow new input
     2036            curdir=pwd;
     2037            cd(DirPBS)
     2038            delete('*')
     2039            cd(curdir)
     2040        else
     2041            [tild,msg1]=mkdir(DirPBS);
     2042            if ~strcmp(msg1,'')
     2043                errormsg=['cannot create ' DirPBS ': ' msg1];%error message for directory creation
     2044                return
     2045            end
     2046        end
     2047        max_walltime=3600*20; % 20h max total calculation (cannot exceed 24 h)
     2048        walltime_onejob=1800; % seconds, max estimated time for asingle file index value
     2049        filename_joblist=fullfile(DirPBS,'job_list.txt');%create name of the global executable file
     2050        fid=fopen(filename_joblist,'w');
     2051        for iprocess=1:length(batch_file_list)
     2052            fprintf(fid,[batch_file_list{iprocess} '\n']);% list of exe files
     2053        end
     2054        fclose(fid);
     2055        system(['chmod +x ' filename_joblist]);% set the file to executable
     2056        pbs_command=['qstat -n CIVX '...
     2057            '-t idempotent --checkpoint ' num2str(walltime_onejob+60) ' '...
     2058            '-l /core=' num2str(NbCore) ','...
     2059            'walltime=' datestr(min(1.05*walltime_onejob/86400*max(NbProcess*BlockLength*nbfield_j,NbCore)/NbCore,max_walltime/86400),13) ' '...
     2060            '-E ' regexprep(filename_joblist,'\.txt\>','.stderr') ' '...
     2061            '-O ' regexprep(filename_joblist,'\.txt\>','.log') ' '...
     2062            extra_oar ' '...
     2063            '"oar-parexec -s -f ' filename_joblist ' '...
     2064            '-l ' filename_joblist '.log"'];
     2065        filename_oarcommand=fullfile(DirPBS,'pbs_command');
     2066        fid=fopen(filename_oarcommand,'w');
     2067        fprintf(fid,pbs_command);
     2068        fclose(fid);
     2069        fprintf(pbs_command);% display in command line
     2070        %system(pbs_command);
     2071        msgbox_uvmat('CONFIRMATION',[ActionName ' command ready to be launched in cluster'])
     2072    case 'python'
     2073        command = [
     2074            'LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | pyp "p.split('':'') | [s for s in p if ''matlab'' not in s] | '':''.join(p)") ' ...
     2075            'python -m fluiddyn.postproc.uvmat ' filexml{iprocess}];
     2076        % fprintf(['command:\n' command '\n\n'])
     2077        system(command, '-echo');
     2078end
     2079
    20352080%------------------------------------------------------------------------
    20362081function STOP_Callback(hObject, eventdata, handles)
  • trunk/src/series/civ_input.m

    r910 r918  
    207207    end
    208208end
    209 if ~isfield(Param.IndexRange,'first_j')||isequal(MaxIndex_j,MinIndex_j)% no possibility of j pairs
    210     set(handles.ListPairMode,'Value',1)
    211     set(handles.ListPairMode,'String',{'series(Di)'})
    212 elseif  MaxIndex_i==1 && MaxIndex_j>1% simple series in j
    213     set(handles.ListPairMode,'String',{'pair j1-j2';'series(Dj)'})
    214     if  MaxIndex_j <= 10
    215         set(handles.ListPairMode,'Value',1)% advice 'pair j1-j2' except in MaxIndex_j is large
    216     end
    217 else
    218     set(handles.ListPairMode,'String',{'pair j1-j2';'series(Dj)';'series(Di)'})%multiple choice
    219     if strcmp(NomTypeNc,'_1-2_1')
    220         set(handles.ListPairMode,'Value',3)% advise 'series(Di)'
    221     elseif  MaxIndex_j <= 10
    222         set(handles.ListPairMode,'Value',1)% advice 'pair j1-j2' except in MaxIndex_j is large
    223     end
    224 end
     209
    225210
    226211%%  transfer the time from the GUI series, or use file index by default
     
    272257%set(handles.CoordUnit,'String',CoordUnit)
    273258set(handles.SearchRange,'UserData', pxcm_search);
    274 
    275 % indicate the min and max indices i and j on the GUI
    276 set(handles.MinIndex_i,'String',num2str(MinIndex_i))
    277 set(handles.MaxIndex_i,'String',num2str(MaxIndex_i))
    278 set(handles.MinIndex_j,'String',num2str(MinIndex_j))
    279 set(handles.MaxIndex_j,'String',num2str(MaxIndex_j))
    280259
    281260
     
    336315    end
    337316end
     317
     318%% set the menu and default choice of civ pairs
     319if ~isfield(Param.IndexRange,'first_j')||isequal(MaxIndex_j,MinIndex_j)% no possibility of j pairs
     320    set(handles.ListPairMode,'Value',1)
     321    set(handles.ListPairMode,'String',{'series(Di)'})
     322elseif  MaxIndex_i==1 && MaxIndex_j>1% simple series in j
     323    set(handles.ListPairMode,'String',{'pair j1-j2';'series(Dj)'})
     324    if  MaxIndex_j <= 10
     325        set(handles.ListPairMode,'Value',1)% advice 'pair j1-j2' except in MaxIndex_j is large
     326    end
     327else
     328    set(handles.ListPairMode,'String',{'pair j1-j2';'series(Dj)';'series(Di)'})%multiple choice
     329    if strcmp(NomTypeNc,'_1-2_1')
     330        set(handles.ListPairMode,'Value',3)% advise 'series(Di)'
     331    elseif  MaxIndex_j <= 10
     332        set(handles.ListPairMode,'Value',1)% advice 'pair j1-j2' except in MaxIndex_j is large
     333    end
     334end
     335
     336%% indicate the min and max indices i and j on the GUI
     337set(handles.MinIndex_i,'String',num2str(MinIndex_i))
     338set(handles.MaxIndex_i,'String',num2str(MaxIndex_i))
     339set(handles.MinIndex_j,'String',num2str(MinIndex_j))
     340set(handles.MaxIndex_j,'String',num2str(MaxIndex_j))
    338341
    339342%% set the reference indices from the input file indices
     
    968971        index_pair=0;
    969972        %get all the Time intervals in bursts
     973        displ_pair_dt='';
    970974        for numod_a=MinIndex_j:MaxIndex_j-1 %nbfield2 always >=2 for 'pair j1-j2' mode
    971975            for numod_b=(numod_a+1):MaxIndex_j
     
    981985           
    982986        end
     987        if index_pair ~=0
    983988        [tild,indsort]=sort(dt);
    984989        displ_pair=displ_pair(indsort);
    985990        displ_pair_dt=displ_pair_dt(indsort);
     991        end
    986992    case 'displacement'
    987993        displ_pair={'Di=Dj=0'};
     
    14971503    end
    14981504end
    1499 set(hObject,'BackgroundColor',[1 0 1])
    15001505set(handles.ConfigSource,'String','NEW')
    15011506set(handles.OK,'BackgroundColor',[1 0 1])
     1507
    15021508%------------------------------------------------------------------------
    15031509% --- Executes on button press in CheckMask: common to all panels (civ1, Civ2..)
     
    15241530        testmask=1;
    15251531    else % browse for a mask
    1526 %         filemask=get(hObject,'UserData');%look for previous mask name stored as UserData
    1527 %         if exist(filemask,'file')
    1528 %             filebase=filemask;
    1529 %         end
    15301532        filemask= uigetfile_uvmat('pick a mask image file:',InputTable{ind_A,1},'image');
    15311533        set(hObject,'UserData',filemask);%store for future use
  • trunk/src/set_grid.m

    r908 r918  
    2424function varargout = set_grid(varargin)
    2525
    26 % Last Modified by GUIDE v2.5 01-Mar-2013 22:41:43
     26% Last Modified by GUIDE v2.5 26-Jun-2015 08:54:56
    2727
    2828% Begin initialization code - DO NOT PLOT
     
    138138UvData=get(huvmat,'UserData');
    139139if isfield(UvData.Field, 'CoordUnit')&& strcmp(UvData.Field.CoordUnit,'pixel')
    140     plot(grid_pix_A(:,1),grid_pix_A(:,2),'.')
    141 else
    142     plot(grid_phys(:,1),grid_phys(:,2),'.')
    143 end
    144 
    145 %% display grid in second image defiend
     140    plot(grid_pix_A(:,1),grid_pix_A(:,2),'.','Tag','proj_object')
     141else
     142    plot(grid_phys(:,1),grid_phys(:,2),'.','Tag','proj_object')
     143end
     144
     145%% display grid in second image defined
    146146if ~isempty(grid_pix_B)
    147147    hviewfield=view_field(get(handles.imageB,'String'));
     
    156156end
    157157
     158% --- Executes on button press in clear.
     159function clear_Callback(hObject, eventdata, handles)
     160huvmat=findobj(allchild(0),'tag','uvmat');
     161if ~isempty(huvmat)
     162    hhuvmat=guidata(huvmat);
     163    hpoints=findobj(hhuvmat.PlotAxes,'Tag','proj_object');
     164    if ~isempty(hpoints)
     165        delete(hpoints)
     166    end
     167end
    158168%------------------------------------------------------------------------
    159169% --- Executes on button press in CoordType.
     
    184194YA=grid_pix_A(:,2);
    185195unitcolumn=32*ones(size(XA));
    186 Xchar=num2str(XA);
     196Xchar=num2str(XA,2);
    187197blanc=char(unitcolumn);
    188 Ychar=num2str(YA);
     198Ychar=num2str(YA,2);
    189199tete=['1 ' num2str(nbpointsA(1))];
    190200txt=[Xchar blanc Ychar];
     
    359369% Hints: get(hObject,'String') returns contents of ImageA as text
    360370%        str2double(get(hObject,'String')) returns contents of ImageA as a double
     371
     372
     373
     374
Note: See TracChangeset for help on using the changeset viewer.